Compare commits
478
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b3b096be8 | ||
|
|
6313f3ea10 | ||
|
|
712d136fe6 | ||
|
|
09ac178888 | ||
|
|
d4600efd7d | ||
|
|
0edd0bca74 | ||
|
|
2fb1dc3214 | ||
|
|
e5ce35cd00 | ||
|
|
e31da597bb | ||
|
|
33675f548a | ||
|
|
008a85b73c | ||
|
|
6fb743f521 | ||
|
|
c0209b3d8e | ||
|
|
c12804f4d9 | ||
|
|
fbb50f8fcd | ||
|
|
0ce3409141 | ||
|
|
46e53a81c2 | ||
|
|
7cf47c04eb | ||
|
|
8855735341 | ||
|
|
c62a501bf0 | ||
|
|
0b03b9c7c5 | ||
|
|
d283efdc9f | ||
|
|
74304052b4 | ||
|
|
2a91caddaf | ||
|
|
78fedabeb1 | ||
|
|
5c20b9ef15 | ||
|
|
fa85de8ef6 | ||
|
|
290e1696b2 | ||
|
|
b7551d2efb | ||
|
|
2e91f0b061 | ||
|
|
61b2b718e6 | ||
|
|
2e23b0b736 | ||
|
|
b81e05387b | ||
|
|
525d7374b2 | ||
|
|
4c4380b94b | ||
|
|
f6fa75e189 | ||
|
|
604f1528d6 | ||
|
|
2aa43ec249 | ||
|
|
fba98ee80f | ||
|
|
511e4b7414 | ||
|
|
4dd7dcfc14 | ||
|
|
d26781b403 | ||
|
|
e2cb398186 | ||
|
|
6a4d88265d | ||
|
|
655692c7ba | ||
|
|
8f9766e46c | ||
|
|
83c1b3ce38 | ||
|
|
000a24d261 | ||
|
|
3228d911cb | ||
|
|
7e358b0f1d | ||
|
|
8d4c13427a | ||
|
|
2049b1a370 | ||
|
|
43d4f511c5 | ||
|
|
f7f3518a04 | ||
|
|
d4aa1fe11b | ||
|
|
14046f9b14 | ||
|
|
be88162cde | ||
|
|
a192052d82 | ||
|
|
1dae2de835 | ||
|
|
0490043990 | ||
|
|
e094e362f4 | ||
|
|
8850a5673e | ||
|
|
cc9c681fa3 | ||
|
|
8f87dbe94d | ||
|
|
7cee3842b8 | ||
|
|
f1c3343929 | ||
|
|
63c8687166 | ||
|
|
b13f65674b | ||
|
|
e6ee939bd4 | ||
|
|
bde60130a8 | ||
|
|
795724f5ca | ||
|
|
44fee9eb0e | ||
|
|
cbcbb9644b | ||
|
|
d1a256b429 | ||
|
|
40bbc7e0d3 | ||
|
|
70f13a3775 | ||
|
|
189e64c486 | ||
|
|
1a1d3a1ecc | ||
|
|
288364f28d | ||
|
|
447bcf3a6b | ||
|
|
07d26adcd3 | ||
|
|
02214cf633 | ||
|
|
f904ff425e | ||
|
|
89370b6f5c | ||
|
|
4edd0e526c | ||
|
|
5d5ee69775 | ||
|
|
9125fcc712 | ||
|
|
17ef1d577d | ||
|
|
da27cd1593 | ||
|
|
aa6a4e7a59 | ||
|
|
8bf45fbafe | ||
|
|
e18be5cabe | ||
|
|
a78fa3db9d | ||
|
|
b81d7475e8 | ||
|
|
7ec6767418 | ||
|
|
30ec4329e0 | ||
|
|
3bae0d8fa2 | ||
|
|
7bd18306d4 | ||
|
|
1302eb9e1f | ||
|
|
f2ed0938ce | ||
|
|
4a84370b2f | ||
|
|
7d1b8864cc | ||
|
|
b18c62f671 | ||
|
|
cc38ca3f8d | ||
|
|
d739516056 | ||
|
|
a6c225eb45 | ||
|
|
5085b681e9 | ||
|
|
9e407f8f76 | ||
|
|
8545e7fc9f | ||
|
|
55a506975b | ||
|
|
b7dfe4c637 | ||
|
|
c4b05a3e93 | ||
|
|
38f2401f17 | ||
|
|
c8b041ab37 | ||
|
|
f52f47e1ff | ||
|
|
213ebe7cfd | ||
|
|
90e278cb97 | ||
|
|
58748c2be1 | ||
|
|
8e9cc58301 | ||
|
|
fb96216e46 | ||
|
|
5059fef3e6 | ||
|
|
564a04b45c | ||
|
|
edf7b3c0bf | ||
|
|
0e819eea4d | ||
|
|
f4ecc6fa64 | ||
|
|
fa918318b2 | ||
|
|
3f2303a086 | ||
|
|
ae6a54bca2 | ||
|
|
d6807233f2 | ||
|
|
3ae308426a | ||
|
|
8a98271680 | ||
|
|
0469641b9d | ||
|
|
4f6f38e16a | ||
|
|
5b6602544e | ||
|
|
efef7183ef | ||
|
|
a4772a5c1d | ||
|
|
26852f4a9d | ||
|
|
82776466b2 | ||
|
|
4837548349 | ||
|
|
b1f262d7d0 | ||
|
|
ddadb75bf4 | ||
|
|
91128d4f0a | ||
|
|
5dbde07b7b | ||
|
|
8a658a4290 | ||
|
|
fb8d2386b9 | ||
|
|
a2558171e1 | ||
|
|
f83835b4b6 | ||
|
|
a313c3e843 | ||
|
|
e7f51caedb | ||
|
|
4b194215dc | ||
|
|
af0fd83aec | ||
|
|
17abe6af32 | ||
|
|
804d5cc065 | ||
|
|
fd62a36041 | ||
|
|
14f68219cf | ||
|
|
7235e4fb92 | ||
|
|
e559798972 | ||
|
|
2dd5239726 | ||
|
|
d409bdf20f | ||
|
|
09791f5a82 | ||
|
|
6298d9d67c | ||
|
|
83391dba1b | ||
|
|
52a1570b3e | ||
|
|
8fe6442c47 | ||
|
|
1fef458c5f | ||
|
|
c104e7500d | ||
|
|
e0b56c83f9 | ||
|
|
d4ac50c4b1 | ||
|
|
f756731700 | ||
|
|
255f8b2aca | ||
|
|
ff58b16cf9 | ||
|
|
788969bcf8 | ||
|
|
ce054dc8dd | ||
|
|
94f555277c | ||
|
|
031cfb8277 | ||
|
|
c1b28cc4fe | ||
|
|
33973d5338 | ||
|
|
f800d672f3 | ||
|
|
98594c9902 | ||
|
|
57193ab2e4 | ||
|
|
f7cd4698db | ||
|
|
180a3fe5db | ||
|
|
4bb1970fef | ||
|
|
ab210fc5ab | ||
|
|
8edc83950c | ||
|
|
8e470f3af4 | ||
|
|
5588875c3f | ||
|
|
a2dd7a5c37 | ||
|
|
7536d9d14e | ||
|
|
9b60eda451 | ||
|
|
541ad1d2e2 | ||
|
|
ce7032c49b | ||
|
|
08ecf10ef3 | ||
|
|
ba502479e8 | ||
|
|
e564eaf75f | ||
|
|
a54d1acf45 | ||
|
|
e82ea89c23 | ||
|
|
06b58bc83b | ||
|
|
17c0734fb2 | ||
|
|
c045057ca0 | ||
|
|
b44f01f89c | ||
|
|
229a3e9855 | ||
|
|
e5192f55ee | ||
|
|
ee2f1efe78 | ||
|
|
8a91871de6 | ||
|
|
64a5074d58 | ||
|
|
7b7796b6fa | ||
|
|
f56eb62d8b | ||
|
|
78f03b7422 | ||
|
|
3aec7f9b02 | ||
|
|
2488d839ab | ||
|
|
2531cd42b6 | ||
|
|
5235bb2e79 | ||
|
|
3b6f3af38e | ||
|
|
15a15cbda5 | ||
|
|
7dbf7a19a1 | ||
|
|
e1fd3eb0f6 | ||
|
|
73280f85c3 | ||
|
|
b8e8981730 | ||
|
|
da787ae360 | ||
|
|
013b7420f6 | ||
|
|
c7fbc530da | ||
|
|
98ad1aa252 | ||
|
|
b01cf63237 | ||
|
|
883d976b5f | ||
|
|
ba4bfea0a4 | ||
|
|
fb7ad5cd73 | ||
|
|
5e931eac19 | ||
|
|
d54eab4052 | ||
|
|
c1b4533201 | ||
|
|
6771db22ae | ||
|
|
d8274204fb | ||
|
|
17c79f25f0 | ||
|
|
d4cbfc1359 | ||
|
|
e0af965d8b | ||
|
|
64d364a768 | ||
|
|
2caaa6fc92 | ||
|
|
fa075dc880 | ||
|
|
f6df07605d | ||
|
|
fc3883b0b6 | ||
|
|
5b2e5bc537 | ||
|
|
25953c6ae1 | ||
|
|
38f89c6976 | ||
|
|
49ea5c96e3 | ||
|
|
7692f201df | ||
|
|
09da8d741e | ||
|
|
b2351a213d | ||
|
|
2abceb1913 | ||
|
|
685810720e | ||
|
|
8e25926e91 | ||
|
|
68628bfd78 | ||
|
|
1dbb7c5740 | ||
|
|
9e3d1d5e68 | ||
|
|
9a70d697af | ||
|
|
a4116a1de7 | ||
|
|
6f721dd536 | ||
|
|
354b9ea396 | ||
|
|
3618831992 | ||
|
|
843a1589d7 | ||
|
|
fc6d5dcbdb | ||
|
|
6d709e1fc1 | ||
|
|
eb11b2c844 | ||
|
|
67e3ebf22f | ||
|
|
9b31746cdb | ||
|
|
ebad58f7e2 | ||
|
|
3c6fb7fc9e | ||
|
|
71aa3e86c6 | ||
|
|
2d8ad2e66a | ||
|
|
1f6f4eda32 | ||
|
|
769815ac0f | ||
|
|
e7888024a8 | ||
|
|
2007a79c45 | ||
|
|
d8286d1790 | ||
|
|
ca74639a74 | ||
|
|
a58a73bb55 | ||
|
|
16594feff1 | ||
|
|
cb46ff8735 | ||
|
|
6f079f13a8 | ||
|
|
a3a0d8f2ce | ||
|
|
5fe8ab11f1 | ||
|
|
b6eb43dd62 | ||
|
|
dc4419b5c5 | ||
|
|
24181c0242 | ||
|
|
db737df597 | ||
|
|
dd9acaaeb3 | ||
|
|
451a4dfe74 | ||
|
|
fb26ca5112 | ||
|
|
389bb96ae4 | ||
|
|
06e668cb90 | ||
|
|
297a3cb549 | ||
|
|
1d0467d4c3 | ||
|
|
9db9499267 | ||
|
|
deda4c985e | ||
|
|
06df6a6167 | ||
|
|
1718d4b1d7 | ||
|
|
16b71ca685 | ||
|
|
5d57b78fe9 | ||
|
|
36332abf35 | ||
|
|
dbbdc95c0b | ||
|
|
fb0be84e24 | ||
|
|
730e44d762 | ||
|
|
f4dd334b49 | ||
|
|
2ac146078f | ||
|
|
726bbf1f1f | ||
|
|
e753009fbc | ||
|
|
f39bc10e3b | ||
|
|
3629bcf057 | ||
|
|
743a3618ea | ||
|
|
77d738d973 | ||
|
|
a3b4ac79af | ||
|
|
c6dd04a7cc | ||
|
|
48cd554f46 | ||
|
|
31bbf207ef | ||
|
|
2b801f5bd4 | ||
|
|
790653dfde | ||
|
|
89be76bb6a | ||
|
|
704fc8d467 | ||
|
|
458e403a04 | ||
|
|
93c0ae19dd | ||
|
|
4e9b076408 | ||
|
|
ae7ec702a5 | ||
|
|
2cccbfd096 | ||
|
|
8f5e7f0c8f | ||
|
|
75d3c40f41 | ||
|
|
63582e44f3 | ||
|
|
4d4743d900 | ||
|
|
289d1e8fa6 | ||
|
|
968b9bf2e6 | ||
|
|
bab2e1391a | ||
|
|
663c9cbc34 | ||
|
|
3d916a685e | ||
|
|
a49fd9cc65 | ||
|
|
3192d89edf | ||
|
|
9edfcc40e9 | ||
|
|
68bdda1a4d | ||
|
|
bb7f2ff365 | ||
|
|
da51481620 | ||
|
|
c93f0d15e4 | ||
|
|
e0cb285993 | ||
|
|
a7d03f7c82 | ||
|
|
ea3bfa5e6e | ||
|
|
ec0b7e2dcc | ||
|
|
feb32600bb | ||
|
|
bc1978ec67 | ||
|
|
24cb379434 | ||
|
|
96be0c3377 | ||
|
|
c38e81c7f5 | ||
|
|
bb63d83f84 | ||
|
|
2915d74822 | ||
|
|
06d4f45e18 | ||
|
|
77dacfbb17 | ||
|
|
d8592d9a4f | ||
|
|
ea4a163187 | ||
|
|
fc92202e04 | ||
|
|
d8da56c6da | ||
|
|
c2b1ba4c5d | ||
|
|
04bb2d1398 | ||
|
|
ad60ba3a46 | ||
|
|
ea3eac51bd | ||
|
|
cacf537004 | ||
|
|
f4a86f5884 | ||
|
|
9f1b227fb8 | ||
|
|
6c79e456a1 | ||
|
|
0b980203fe | ||
|
|
73bf290f58 | ||
|
|
475bb4bb0b | ||
|
|
efadc691e7 | ||
|
|
4af36c69a7 | ||
|
|
bd107d2015 | ||
|
|
f007ca4227 | ||
|
|
50c4674dfa | ||
|
|
c12b870104 | ||
|
|
92aedb9365 | ||
|
|
368204115c | ||
|
|
971cdb44e2 | ||
|
|
16e4e61a0f | ||
|
|
4b36cc29a6 | ||
|
|
31ddd09087 | ||
|
|
68173de8bd | ||
|
|
960e3309bc | ||
|
|
2ee7589474 | ||
|
|
297992a4ae | ||
|
|
3f85230417 | ||
|
|
43c3158bc6 | ||
|
|
d95e9f3229 | ||
|
|
d581f9659a | ||
|
|
ea94f9234c | ||
|
|
c19468b04e | ||
|
|
c47afd8f4b | ||
|
|
afd5a3ba07 | ||
|
|
7dbad4c6d2 | ||
|
|
083a5042f6 | ||
|
|
351ca7c290 | ||
|
|
f907be2c14 | ||
|
|
6201f58c20 | ||
|
|
8fee764c73 | ||
|
|
3dd94f714d | ||
|
|
c716670be0 | ||
|
|
352066dde6 | ||
|
|
d6bd31a166 | ||
|
|
c0a4b139a2 | ||
|
|
6d60953f98 | ||
|
|
17ed06ee4d | ||
|
|
9427dac823 | ||
|
|
d74e082155 | ||
|
|
2e0013ec25 | ||
|
|
2493aa6529 | ||
|
|
85c89b16ef | ||
|
|
4f75925410 | ||
|
|
820c458f4c | ||
|
|
dc9d65524d | ||
|
|
1a4c788ed7 | ||
|
|
df8b9ac3b1 | ||
|
|
fcb1e40582 | ||
|
|
4973372612 | ||
|
|
3f6c085e8a | ||
|
|
040e3097b5 | ||
|
|
feef96ed16 | ||
|
|
4f722899df | ||
|
|
166a918b43 | ||
|
|
1b4c860f63 | ||
|
|
3b6945b2db | ||
|
|
4174e23c04 | ||
|
|
3993175e45 | ||
|
|
60c7ffc981 | ||
|
|
33de9dde4f | ||
|
|
6b85220fa4 | ||
|
|
1cbae7d41c | ||
|
|
b3a4b09f5b | ||
|
|
c8b26735c8 | ||
|
|
907e092abc | ||
|
|
3073495a06 | ||
|
|
16c411e436 | ||
|
|
2d841870a8 | ||
|
|
7f8ce7f5c0 | ||
|
|
07b5f13474 | ||
|
|
f4aac599ca | ||
|
|
2d63f0ba62 | ||
|
|
1d41682715 | ||
|
|
705765d113 | ||
|
|
16a97ed65d | ||
|
|
ecf532af0e | ||
|
|
ce600df98a | ||
|
|
b4a6a34705 | ||
|
|
a7e51b416b | ||
|
|
71f786f146 | ||
|
|
848c6fdc56 | ||
|
|
ead9ee2d13 | ||
|
|
4861d1f78d | ||
|
|
5c76c9d3ac | ||
|
|
699cbfb7c1 | ||
|
|
319d56be7e | ||
|
|
11d3678984 | ||
|
|
f55f150896 | ||
|
|
e1f1b5d18b | ||
|
|
f0fe454145 | ||
|
|
4026f6c188 | ||
|
|
29cfb8c177 | ||
|
|
a092ee9ff9 | ||
|
|
6c3cbee35c | ||
|
|
4b23db28b3 | ||
|
|
646f83642a | ||
|
|
d9bfba1e62 | ||
|
|
f1bb7747ae | ||
|
|
d2eb3e5ecf | ||
|
|
b0b1cf4f01 | ||
|
|
d91ea41f64 | ||
|
|
5fed429827 | ||
|
|
9a569bfb8d | ||
|
|
e2c41ffa5a | ||
|
|
13e4ed8515 | ||
|
|
c66fe11e16 | ||
|
|
75d0eb32be | ||
|
|
c44b9f91cd | ||
|
|
82da252cf0 | ||
|
|
ce72b67cef | ||
|
|
2a94d16834 | ||
|
|
da7baa5e39 |
@@ -1 +1,4 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// AssignerModule is the asker the assigner's compose asks name.
|
||||
const AssignerModule = "mesh-delivery-assigner"
|
||||
|
||||
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
|
||||
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
|
||||
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
|
||||
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
|
||||
// second ask builds nothing.
|
||||
type composeAsks struct {
|
||||
ctx context.Context
|
||||
asker buildrun.Asker
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
asks map[string]*composeAsk
|
||||
}
|
||||
|
||||
type composeAsk struct {
|
||||
id string
|
||||
cancel context.CancelFunc
|
||||
ran *assign.Ran
|
||||
}
|
||||
|
||||
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
|
||||
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
|
||||
}
|
||||
|
||||
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
|
||||
// A job its layout refuses ends at once, failed with why: never silent.
|
||||
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if k, ok := a.asks[key]; ok {
|
||||
return k.ran, k.id
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
|
||||
At: a.now(), Compose: &job}
|
||||
ctx, cancel := context.WithCancel(a.ctx)
|
||||
k := &composeAsk{id: ask.ID, cancel: cancel}
|
||||
a.asks[key] = k
|
||||
if err := a.asker.Put(ctx, ask); err != nil {
|
||||
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
|
||||
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
|
||||
cancel()
|
||||
return k.ran, k.id
|
||||
}
|
||||
go func() {
|
||||
step, err := a.asker.Wait(ctx, ask)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
ran := assign.RanOf(step)
|
||||
if err != nil {
|
||||
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
|
||||
}
|
||||
a.mu.Lock()
|
||||
k.ran = &ran
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}()
|
||||
return nil, k.id
|
||||
}
|
||||
|
||||
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
|
||||
func (a *composeAsks) sweep(seen map[string]bool) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for key, k := range a.asks {
|
||||
if !seen[key] {
|
||||
k.cancel()
|
||||
delete(a.asks, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
|
||||
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
|
||||
// asks. The keys it used are marked in seen.
|
||||
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
|
||||
waiting := func(words string) delivery.CompositionVerdict {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
|
||||
}
|
||||
head, err := assign.HeadJob(m, ch)
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
|
||||
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
|
||||
}
|
||||
var pending []string
|
||||
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
|
||||
seen[key] = true
|
||||
headRan, id := p.asks.get(key, head)
|
||||
if headRan == nil {
|
||||
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
|
||||
}
|
||||
var readers []assign.Reader
|
||||
var notJudged []string
|
||||
for _, module := range sortedNames(assign.ReaderModules) {
|
||||
repository := assign.ReaderModules[module]
|
||||
var commits []string
|
||||
member := false
|
||||
if h := ch.Members[repository]; h != "" {
|
||||
commits, member = []string{h}, true
|
||||
} else {
|
||||
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
|
||||
"commit on record, and the group holds no member of "+repository)
|
||||
continue
|
||||
}
|
||||
for _, commit := range commits {
|
||||
job, err := assign.ReaderJob(ch, module, repository, commit)
|
||||
if err != nil {
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
|
||||
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
|
||||
continue
|
||||
}
|
||||
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
|
||||
seen[rkey] = true
|
||||
ran, rid := p.asks.get(rkey, job)
|
||||
if ran == nil {
|
||||
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
|
||||
continue
|
||||
}
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
|
||||
}
|
||||
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
|
||||
v.Warnings = append(v.Warnings, notJudged...)
|
||||
return v
|
||||
}
|
||||
|
||||
func sortedNames(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
-180
@@ -1,180 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
|
||||
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
|
||||
type holder struct {
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
}
|
||||
|
||||
func (h *holder) taken() []delivery.StepAsk {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), h.asks...)
|
||||
}
|
||||
|
||||
func (b *bed) hold(t *testing.T) *holder {
|
||||
t.Helper()
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
|
||||
h := &holder{}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
a, err := delivery.DecodeStepAsk(m.Data())
|
||||
if err != nil || a.Compose == nil {
|
||||
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
|
||||
m.Ack()
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.asks = append(h.asks, a)
|
||||
h.mu.Unlock()
|
||||
req, err := composer.Decode(a.Compose.Request)
|
||||
if err == nil {
|
||||
err = composer.Fill(&req, func(digest string) ([]byte, error) {
|
||||
key, _ := delivery.ManifestsKey(digest)
|
||||
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
man, err := delivery.DecodeManifest(e.Value())
|
||||
return []byte(man.JSON), err
|
||||
})
|
||||
}
|
||||
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else if v, err := composer.Run(req); err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else {
|
||||
vr, _ := json.Marshal(v)
|
||||
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
|
||||
}
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
t.Errorf("the step cannot be written: %v", err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
if _, err := steps.Put(b.ctx, key, raw); err != nil {
|
||||
t.Errorf("the step %s: %v", key, err)
|
||||
}
|
||||
m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(cc.Stop)
|
||||
return h
|
||||
}
|
||||
|
||||
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
|
||||
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
|
||||
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// head's composer was built with, and asks nothing more on a later pass of the same heads.
|
||||
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
running := strings.Repeat("3", 40)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
|
||||
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
b.start(t)
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
// No holder takes the asks yet: the composition waits, naming them.
|
||||
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
|
||||
}
|
||||
h := b.hold(t)
|
||||
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
|
||||
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14", v)
|
||||
}
|
||||
asks := h.taken()
|
||||
if len(asks) != 2 {
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
|
||||
}
|
||||
byRepo := map[string]delivery.StepAsk{}
|
||||
for _, a := range asks {
|
||||
byRepo[a.Compose.Fetch["repository"].(string)] = a
|
||||
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
|
||||
t.Errorf("ask = %+v", a)
|
||||
}
|
||||
}
|
||||
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
|
||||
t.Errorf("the head's ask = %+v", a.Compose)
|
||||
}
|
||||
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running controller's ask = %+v", a.Compose)
|
||||
}
|
||||
// A later pass of the same heads (woken by any change) asks nothing more.
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := len(h.taken()); n != 2 {
|
||||
t.Errorf("a later pass of the same heads asked again: %d asks", n)
|
||||
}
|
||||
}
|
||||
|
||||
// verdict waits for the composition under key to satisfy ok, and answers it.
|
||||
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
|
||||
t.Helper()
|
||||
var last delivery.CompositionVerdict
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
|
||||
return last
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
|
||||
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
|
||||
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
|
||||
// where they part. The assigner composes nothing and never writes these.
|
||||
|
||||
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
|
||||
// it was refused or cannot be read.
|
||||
type publishedRow struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Unread string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
|
||||
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
|
||||
// A value or key that does not read is named with why, never left out.
|
||||
func published(values map[string][]byte) map[string][]publishedRow {
|
||||
byNode := map[string]map[string][]byte{}
|
||||
for key, raw := range values {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
if byNode[node] == nil {
|
||||
byNode[node] = map[string][]byte{}
|
||||
}
|
||||
byNode[node][key] = raw
|
||||
}
|
||||
out := map[string][]publishedRow{}
|
||||
for node, keys := range byNode {
|
||||
var rows []publishedRow
|
||||
for key, raw := range keys {
|
||||
row := publishedRow{Key: key}
|
||||
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
row.Unread = err.Error()
|
||||
case isNode:
|
||||
row.Module = "node"
|
||||
default:
|
||||
row.Module, row.FP = module, fp
|
||||
}
|
||||
if err == nil {
|
||||
c, derr := delivery.DecodeComposed(raw)
|
||||
switch {
|
||||
case derr != nil:
|
||||
row.Unread = derr.Error()
|
||||
case c.Refused != "":
|
||||
row.Refused = c.Refused
|
||||
default:
|
||||
if d, err := delivery.ComposedDigestOf(c); err != nil {
|
||||
row.Unread = err.Error()
|
||||
} else {
|
||||
row.Digest = d
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
|
||||
out[node] = rows
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -17,12 +19,15 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -35,6 +40,8 @@ type bed struct {
|
||||
ctx context.Context
|
||||
dir string
|
||||
buckets map[string]jetstream.KeyValue
|
||||
// placed is, by node, the modules the bed's placement names on it (place).
|
||||
placed map[string][]string
|
||||
}
|
||||
|
||||
func startBed(t *testing.T, leaveOut string) *bed {
|
||||
@@ -69,12 +76,21 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
if name == leaveOut {
|
||||
continue
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
|
||||
if name == delivery.PlacementPulsesBucket {
|
||||
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.buckets[name] = kv
|
||||
}
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog),
|
||||
Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -128,9 +144,7 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
|
||||
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
|
||||
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
done := b.start(t)
|
||||
func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
|
||||
events, err := b.js.Stream(b.ctx, "EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -155,7 +169,7 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -172,6 +186,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
|
||||
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
for _, missing := range readBuckets {
|
||||
if startsWithout(missing) {
|
||||
continue // it starts without it (the placement and the grants' source: main_test.go; the puts: below)
|
||||
}
|
||||
b := startBed(t, missing)
|
||||
select {
|
||||
case err := <-b.start(t):
|
||||
@@ -218,6 +235,28 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
t.Fatal("the put was not taken further once its bucket was made")
|
||||
}
|
||||
}
|
||||
|
||||
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
|
||||
// it from the manifest) it is read with no restart: a node it names gets its own want, the node composing its own.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
select {
|
||||
case err := <-pdone:
|
||||
t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
plkey, _ := delivery.LegacyPlacementKey("n1") // the form before, read beside the new (research 072 row 37)
|
||||
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
|
||||
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
|
||||
}
|
||||
}
|
||||
|
||||
// putJSON writes a value into one of the bed's buckets.
|
||||
@@ -228,24 +267,43 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
|
||||
// move to it, in mesh-controller_composed.
|
||||
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
|
||||
// place adds module to the seat's placement of node, written as the seat writes it: one node key and a key per module
|
||||
// (research 072 row 37). The placement alone names where modules run (ADR 0567 §8).
|
||||
func (b *bed) place(t *testing.T, node, module string) {
|
||||
t.Helper()
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if b.placed == nil {
|
||||
b.placed = map[string][]string{}
|
||||
}
|
||||
if !slices.Contains(b.placed[node], module) {
|
||||
b.placed[node] = append(b.placed[node], module)
|
||||
}
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range b.placed[node] {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
b.putJSON(t, delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
|
||||
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
|
||||
func (b *bed) compose(t *testing.T, node string, d byte) string {
|
||||
t.Helper()
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat(string(d), 64)
|
||||
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
|
||||
if judgement != delivery.JudgedNeither {
|
||||
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(cd)
|
||||
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
b.putJSON(t, delivery.HaveBucket, key, v)
|
||||
return digest
|
||||
}
|
||||
|
||||
@@ -262,62 +320,6 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||
return digest
|
||||
}
|
||||
|
||||
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
|
||||
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
|
||||
func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
|
||||
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
|
||||
pulls := []delivery.CandidatePull{{Head: heads[0],
|
||||
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
began := time.Now()
|
||||
b.start(t)
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
var v delivery.CompositionVerdict
|
||||
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("no composition under %s after 10 s", key)
|
||||
}
|
||||
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if v, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14 alone naming its mesh-sdk", v)
|
||||
}
|
||||
if took := time.Since(began); took > 5*time.Second {
|
||||
t.Errorf("the composition took %s", took)
|
||||
}
|
||||
// 1622/AC3: a group without a mesh-sdk member builds no composer: no ask is put on the Go build seat.
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
if st, err := b.js.Stream(b.ctx, seat); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
|
||||
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// get waits for a key of one of the bed's buckets and answers its value.
|
||||
func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
||||
t.Helper()
|
||||
@@ -349,20 +351,20 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
for _, fp := range []string{fNew, fOld} {
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.compose(t, node, 'c')
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||
b.place(t, node, "web")
|
||||
}
|
||||
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.compose(t, "shanks", 'a')
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -370,18 +372,18 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
|
||||
}
|
||||
var neither asks.Condition
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
|
||||
if strings.HasSuffix(k, ".neither") && c.State == asks.StateRaised {
|
||||
neither = c
|
||||
}
|
||||
}
|
||||
}
|
||||
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
|
||||
}
|
||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||
@@ -395,8 +397,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
// there by itself: the condition is cleared.
|
||||
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F, the node composing it", w)
|
||||
}
|
||||
cleared := false
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||
@@ -406,6 +408,32 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if !cleared {
|
||||
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
|
||||
// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
|
||||
// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.LegacyPlacementKey(node)
|
||||
b.putJSON(t, delivery.PlacementBucket, pkey, pl)
|
||||
}
|
||||
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
|
||||
t.Errorf("ace's want once placed = %+v", w)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
var self bool
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
nw, err := delivery.DecodeNodeWant(e.Value())
|
||||
self = err == nil && nw.Self
|
||||
}
|
||||
}
|
||||
if !self {
|
||||
t.Error("ace's own want does not say it composes its own once placed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
|
||||
@@ -421,7 +449,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.place(t, "ace", "web")
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||
@@ -436,36 +464,34 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
|
||||
// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
|
||||
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
|
||||
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
|
||||
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
|
||||
t.Helper()
|
||||
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
|
||||
w, err := b.buckets[assign.FindingsBucket].WatchAll(b.ctx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
out := map[string]asks.Condition{}
|
||||
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
return out
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
c, err := asks.DecodeCondition(m.Data())
|
||||
c := out[e.Key()]
|
||||
c.Key, c.State = e.Key(), asks.StateCleared
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[c.Key] = c
|
||||
}
|
||||
if n == 0 {
|
||||
return out
|
||||
c = asks.Condition{Key: e.Key(), State: asks.StateRaised, Severity: string(f.Severity), Explanation: f.Explanation,
|
||||
Resolved: f.Resolved, Subject: asks.Scope{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
|
||||
}
|
||||
out[e.Key()] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
|
||||
@@ -484,7 +510,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
wantKey, _ := delivery.WantKey("ace", "mail")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
|
||||
b.place(t, "ace", "mail")
|
||||
b.start(t)
|
||||
|
||||
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
@@ -498,7 +524,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
key := ""
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
|
||||
if c.State == asks.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") {
|
||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||
}
|
||||
@@ -533,11 +559,11 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
|
||||
gkey, _ := delivery.PartGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", 'c')
|
||||
b.place(t, "ace", name)
|
||||
}
|
||||
count := func() int {
|
||||
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
|
||||
@@ -567,58 +593,6 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
|
||||
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
|
||||
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
|
||||
func TestAWantWaitsForItsComposition(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
time.Sleep(time.Second)
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
t.Fatalf("a want was written before the composition existed: %s", e.Value())
|
||||
}
|
||||
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
|
||||
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
|
||||
}
|
||||
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
|
||||
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
|
||||
}
|
||||
// the node's own want
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
|
||||
}
|
||||
ownKey, _ := delivery.ComposedNodeKey("ace")
|
||||
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
|
||||
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the node's own want = %+v, %v", w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
|
||||
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
t.Helper()
|
||||
@@ -628,12 +602,12 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
gkey, _ := delivery.PartGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
|
||||
b.place(t, "ace", name)
|
||||
}
|
||||
|
||||
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
|
||||
@@ -696,8 +670,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.compose(t, "ace", 'a')
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
@@ -712,13 +686,13 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b.get(t, delivery.AssignerAsksBucket, hold)
|
||||
b.answer(t, hold, "release")
|
||||
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
|
||||
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
if w.Composed != "" || !w.Self || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
|
||||
cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held"))
|
||||
}
|
||||
if cleared {
|
||||
return
|
||||
@@ -743,19 +717,19 @@ func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk
|
||||
|
||||
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
|
||||
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
|
||||
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
|
||||
// value, so the node retries.
|
||||
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
|
||||
b.compose(t, "ace", 'b')
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
|
||||
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
|
||||
b.start(t)
|
||||
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
|
||||
a := b.askNow(t, akey, 0)
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") {
|
||||
t.Fatalf("the backup ask = %+v", a)
|
||||
}
|
||||
// an answer to another ask is no answer to this one
|
||||
@@ -766,7 +740,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
|
||||
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
|
||||
if w1.FP != fNew || w1.Composed != first {
|
||||
if w1.FP != fNew || w1.Composed != "" || !w1.Self {
|
||||
t.Errorf("the want = %+v", w1)
|
||||
}
|
||||
// the node retries and the backup fails again: a new have value, asked again under the next number
|
||||
@@ -781,91 +755,10 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
if w2.Words == w1.Words {
|
||||
t.Errorf("the second answer left the want as it was: %+v", w2)
|
||||
}
|
||||
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
|
||||
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
|
||||
a3 := b.askNow(t, akey, 3)
|
||||
if !strings.Contains(a3.Ask.Explanation, second) {
|
||||
t.Errorf("the ask for the new revision = %+v", a3)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Errorf("the old answer was taken for the new revision: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
|
||||
|
||||
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
|
||||
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
|
||||
// neither is written at once.
|
||||
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, recorded)
|
||||
old := "sha256:" + strings.Repeat("a", 64)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
|
||||
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
|
||||
b.start(t)
|
||||
akey, _ := delivery.RecordAskKey("ace", "web", nw)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
|
||||
t.Fatalf("the record ask = %+v, %v", a, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the move was written before the yes: %+v", w)
|
||||
}
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
|
||||
// a new revision at the same fingerprint, judged to move data, waits as well
|
||||
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
|
||||
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
|
||||
b.get(t, delivery.AssignerAsksBucket, rkey)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
|
||||
t.Fatalf("the revision was written before the yes: %+v", w)
|
||||
}
|
||||
// and a revision judged neither is written at once
|
||||
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
|
||||
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
|
||||
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
|
||||
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
|
||||
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
|
||||
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
|
||||
}
|
||||
// the controller composes the bus at the fingerprint it runs with a new user list
|
||||
key, _ := delivery.ComposedKey("ace", "bus", fOld)
|
||||
free := "sha256:" + strings.Repeat("e", 64)
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
|
||||
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
if w.FP != fOld {
|
||||
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
|
||||
}
|
||||
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
|
||||
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||
// cleared saying why, and the want stays at the fingerprint.
|
||||
@@ -873,7 +766,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
@@ -889,7 +782,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
if strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held") {
|
||||
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
@@ -906,7 +799,7 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
@@ -943,3 +836,278 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
|
||||
// their local names, another module's `<module>.<state>`.
|
||||
type bedStore struct{ b *bed }
|
||||
|
||||
func (s bedStore) kv(state string) jetstream.KeyValue {
|
||||
if strings.Contains(state, ".") {
|
||||
return s.b.buckets[strings.Replace(state, ".", "_", 1)]
|
||||
}
|
||||
return s.b.buckets[delivery.AssignerModule+"_"+state]
|
||||
}
|
||||
|
||||
func (s bedStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.kv(state).Get(s.b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (s bedStore) put(state, key string, value []byte) error {
|
||||
_, err := s.kv(state).Put(s.b.ctx, key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s bedStore) keys(state string) ([]string, error) {
|
||||
keys, err := s.kv(state).Keys(s.b.ctx)
|
||||
if err != nil && strings.Contains(err.Error(), "no keys") {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (s bedStore) del(state, key string) error { return s.kv(state).Delete(s.b.ctx, key) }
|
||||
|
||||
func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { return nil, nil }
|
||||
|
||||
// 1126 (ADR 0567 §2-§6), the seat mesh-placement whole on the bus: a proposal moving web from the anchor to the laptop is
|
||||
// proposed through the verb, the assigner finds both nodes must answer, they answer the same in their haves, the
|
||||
// operator is asked and approves, and each key is written at a new revision while placement-changed is heard on the
|
||||
// assigner's stream naming both nodes. A second proposal laid over the old revision of a key moved since is laid again,
|
||||
// and declined, writes nothing and is withdrawn.
|
||||
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + delivery.AssignerModule + "_placement",
|
||||
Subjects: []string{process.ModuleStreamSubject(delivery.AssignerModule, ">")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := make(chan delivery.PlacementEvent, 16)
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
if _, err := nc.Subscribe(process.ModuleStreamSubject(delivery.AssignerModule, delivery.PlacementChangedEvent), func(m *nats.Msg) {
|
||||
var e delivery.PlacementEvent
|
||||
if json.Unmarshal(m.Data, &e) == nil {
|
||||
heard <- e
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
|
||||
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
|
||||
// place writes a node's placement as another writer would, in its node and module keys (research 072 row 37), and
|
||||
// answers its node key's revision; legacy writes it in the form before, placement.<node>, alone.
|
||||
place := func(node string, legacy bool, modules ...string) uint64 {
|
||||
p := delivery.Placement{Node: node, Modules: []delivery.PlacedModule{}}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, _ := delivery.PlacementValues(p)
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
if legacy {
|
||||
key, _ = delivery.LegacyPlacementKey(node)
|
||||
values = map[string][]byte{key: values[key]}
|
||||
}
|
||||
for _, k := range slices.Sorted(maps.Keys(values)) {
|
||||
if k == key {
|
||||
continue
|
||||
}
|
||||
if _, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, k, values[k]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rev, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, key, values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
place("anchor", true, "web")
|
||||
laptopRev := place("laptop", false, "shell")
|
||||
bound := judge.AnswerBound
|
||||
t.Cleanup(func() { judge.AnswerBound = bound })
|
||||
judge.AnswerBound = 2 * time.Second
|
||||
saysWhatItKeeps(t, b, b.start(t))
|
||||
// The holder converts anchor, held in the form before alone, to its node and module keys once at its start.
|
||||
var anchorRev uint64
|
||||
for deadline := time.Now().Add(15 * time.Second); anchorRev == 0; time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.node"); err == nil {
|
||||
anchorRev = e.Revision()
|
||||
if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.web"); err != nil {
|
||||
t.Fatalf("anchor converted without its module key: %v", err)
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("anchor in the form before was not converted")
|
||||
}
|
||||
}
|
||||
verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
|
||||
propose := func() delivery.PlacementProposal {
|
||||
t.Helper()
|
||||
got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
|
||||
`{"op":"place","node":"laptop","module":"web"}]`}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got.(proposed).Proposal
|
||||
}
|
||||
proposalNow := func(id, what string, ok func(delivery.PlacementProposal) bool) delivery.PlacementProposal {
|
||||
t.Helper()
|
||||
var last delivery.PlacementProposal
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.PlacementProposalsBucket].Get(b.ctx, id); err == nil {
|
||||
if p, err := delivery.DecodePlacementProposal(e.Value()); err == nil {
|
||||
if last = p; ok(p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the proposal %s never was %s: %+v", id, what, last)
|
||||
return last
|
||||
}
|
||||
answerAll := func(p delivery.PlacementProposal) {
|
||||
t.Helper()
|
||||
for _, node := range p.Must {
|
||||
key, _ := delivery.PlacementCandidateKey(p.ID, node)
|
||||
e, err := b.buckets[delivery.PlacementCandidatesBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("no candidate %s: %v", key, err)
|
||||
}
|
||||
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
|
||||
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".node", have)
|
||||
}
|
||||
}
|
||||
|
||||
p := propose()
|
||||
p = proposalNow(p.ID, "told who must answer", func(p delivery.PlacementProposal) bool { return len(p.Must) > 0 })
|
||||
if !slices.Equal(p.Must, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("must answer = %v, want both nodes", p.Must)
|
||||
}
|
||||
answerAll(p)
|
||||
p = proposalNow(p.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
|
||||
akey, _ := delivery.PlacementAskKey(p.ID)
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Proposal.ID != p.ID || len(e.Nodes) != 2 || e.Nodes[0].Before != anchorRev || e.Nodes[1].Before != laptopRev ||
|
||||
e.Nodes[0].After <= anchorRev || e.Warrant == nil || !slices.Equal(e.Nodes[0].Removed, []string{"web"}) {
|
||||
t.Errorf("placement-changed = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard")
|
||||
}
|
||||
for node, want := range map[string][]string{"anchor": {}, "laptop": {"shell", "web"}} {
|
||||
// The node key names the modules, each module key carries its own, and the mirror of the form before follows.
|
||||
values := map[string][]byte{}
|
||||
for _, key := range []string{node + ".node", "placement." + node, node + ".web", node + ".shell"} {
|
||||
if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); err == nil {
|
||||
values[key] = e.Value()
|
||||
}
|
||||
}
|
||||
got, err := delivery.PlacementsOf(values)
|
||||
if err != nil || !slices.Equal(got[node].Assigned(), want) {
|
||||
t.Errorf("%s = %v, %v, want %v", node, got[node].Assigned(), err, want)
|
||||
}
|
||||
mirror, _ := delivery.DecodePlacement(values["placement."+node])
|
||||
if !slices.Equal(mirror.Assigned(), want) {
|
||||
t.Errorf("the mirror placement.%s = %v, want %v", node, mirror.Assigned(), want)
|
||||
}
|
||||
}
|
||||
if b.hasKey(delivery.PlacementProposalsBucket, p.ID) {
|
||||
t.Error("an applied proposal stays in the state")
|
||||
}
|
||||
|
||||
// A proposal whose key moves before its warrant is laid again; then declined, it writes nothing and ends.
|
||||
q, err := verbs.doPropose(map[string]any{"why": "back", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := q.(proposed).Proposal
|
||||
place("laptop", false, "shell", "web") // another writer moved the key
|
||||
relaid := proposalNow(moved.ID, "laid again", func(p delivery.PlacementProposal) bool {
|
||||
return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
|
||||
})
|
||||
answerAll(relaid)
|
||||
proposalNow(moved.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
|
||||
akey, _ = delivery.PlacementAskKey(moved.ID)
|
||||
b.answer(t, akey, assign.OptionNo)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.PlacementProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("a declined proposal was not ended")
|
||||
}
|
||||
}
|
||||
key, _ := delivery.PlacementNodeKey("laptop")
|
||||
if e, _ := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); e == nil || !strings.Contains(string(e.Value()), "web") {
|
||||
t.Error("a declined proposal wrote its key")
|
||||
}
|
||||
|
||||
// ADR 0593 §8: a node that does not answer is named as giving no answer once its pulse key's time to live ends, heard on
|
||||
// the bus with no timer of the assigner's, and the proposal is decided on the answers it has: asked, not failed.
|
||||
q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
silent := q.(proposed).Proposal
|
||||
decided := proposalNow(silent.ID, "decided without its silent nodes", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
|
||||
if decided.Verdict == nil || decided.Verdict.State != delivery.VerdictPass ||
|
||||
!strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "no answer from") || !strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "laptop (offline)") {
|
||||
t.Errorf("the silent node's verdict = %+v", decided.Verdict)
|
||||
}
|
||||
|
||||
// 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised,
|
||||
// and placement-changed carries that warrant. The assigner reads no lease (ADR 0567 §8): the enrolment's own issue waits
|
||||
// for the lease write (ADR 0587 §4).
|
||||
for len(heard) > 0 {
|
||||
<-heard
|
||||
}
|
||||
joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
|
||||
"changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Proposal.ID != joined.(proposed).Proposal.ID || e.Warrant == nil || e.Warrant.Warrant != "enrol-ace-1@2026-10-28T20:00:00Z" || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
|
||||
t.Errorf("placement-changed for the joining node = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard for the joining node")
|
||||
}
|
||||
if ace, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") {
|
||||
t.Errorf("the joining node's key = %v, %v", ace, err)
|
||||
}
|
||||
if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) {
|
||||
t.Error("an ask was raised for the joining node, or its proposal stays open")
|
||||
}
|
||||
// Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and
|
||||
// placement-changed carries the warrant (the operator's decision of 2026-11-02 01:05).
|
||||
for len(heard) > 0 {
|
||||
<-heard
|
||||
}
|
||||
if _, err := verbs.doPropose(map[string]any{"why": "enrolment of ace revoked", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
|
||||
"changes": `[{"op":"remove-node","node":"ace"}]`}, "novox.mesh-enrolment"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Warrant == nil || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
|
||||
t.Errorf("placement-changed for the removed node = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard for the removed node")
|
||||
}
|
||||
if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err == nil {
|
||||
t.Error("the removed node's key stays")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"slices"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -24,13 +24,15 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
if os.Getenv(process.BrokerFileEnv) == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
@@ -50,37 +52,48 @@ func main() {
|
||||
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
provisions.ProvisionsBucket,
|
||||
}
|
||||
|
||||
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
|
||||
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
|
||||
var readFamilies = []string{delivery.ComposedRead}
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
|
||||
// (ADR 0567 §1), named after the assigner as every module state is.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket}
|
||||
|
||||
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
|
||||
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
|
||||
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||
// (and the node agent from release 6), its differences by the view, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
|
||||
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
{Name: delivery.PlacementCandidatesState, Readable: true},
|
||||
{Name: delivery.PlacementPulsesState, Readable: true},
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
{Name: health.FindingsState, Readable: true},
|
||||
{Name: delivery.AssignerPoliciesState, Readable: true},
|
||||
{Name: delivery.SeatHoldersState, Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
@@ -98,6 +111,40 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.SeatHoldersBucket && err != nil {
|
||||
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
|
||||
// holds which seat here, and the controller's seats still answer.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == provisions.ProvisionsBucket && err != nil {
|
||||
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
|
||||
// until then no want waits for a provision, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.PoliciesBucket && err != nil {
|
||||
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
|
||||
// person's policy is read, so the controller's beside each composition holds, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.FindingsBucket && err != nil {
|
||||
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
|
||||
// router and kept as no finding.
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
@@ -106,36 +153,29 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's
|
||||
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
|
||||
delivery.AssignerPutsBucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||
}
|
||||
account, err := process.ReadAccount(path)
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(account.Node + "." + account.Module),
|
||||
nats.UserInfo(account.User, account.Password),
|
||||
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(account.URL, opts...)
|
||||
conn, js, err := account.Dial()
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -150,10 +190,33 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := said.Keeps(keptStates...); err != nil {
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
thisNode = account.Node
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
findings := findingsOf(ctx, kvs, logger.Printf)
|
||||
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger)
|
||||
}
|
||||
|
||||
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
|
||||
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
|
||||
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
|
||||
var f assign.Findings
|
||||
if kv := kvs[assign.FindingsBucket]; kv != nil {
|
||||
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
|
||||
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
|
||||
f.Keys = func() ([]string, error) {
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
|
||||
var thisNode string
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
@@ -162,24 +225,27 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
delivery.AssignerCompositionsBucket,
|
||||
delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerWantsBucket,
|
||||
assign.PoliciesBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SeatHoldersBucket,
|
||||
}
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -223,7 +289,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
w.put(name, e.Key(), value)
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -263,7 +329,12 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && startsWithout(name) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
@@ -275,7 +346,37 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
for range len(watched) + 1 {
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
|
||||
// module is placed.
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets)+2)
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
}
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
}
|
||||
for range started + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -284,11 +385,16 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -297,8 +403,29 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
|
||||
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
|
||||
// bus now holds, and says the seat waits while one is missing.
|
||||
var unbound []string
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] != nil || js == nil {
|
||||
continue
|
||||
}
|
||||
if kv, err := js.KeyValue(ctx, name); err == nil {
|
||||
if err := takePlacement(appeared{name, kv}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
unbound = append(unbound, name)
|
||||
}
|
||||
p.sayWaits(unbound)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -24,18 +26,30 @@ import (
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reads []string
|
||||
for _, r := range m.Reads {
|
||||
if slices.Contains(readFamilies, r) {
|
||||
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
|
||||
}
|
||||
reads = append(reads, strings.Replace(r, ".", "_", 1))
|
||||
}
|
||||
for _, l := range readFamilies {
|
||||
if !slices.Contains(m.Reads, l) {
|
||||
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
for _, s := range m.State {
|
||||
states = append(states, m.Module+"_"+s.Name)
|
||||
@@ -62,16 +76,19 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
@@ -86,15 +103,17 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[delivery.PlacementBucket].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
@@ -105,14 +124,6 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
func TestNoCredential(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
@@ -170,3 +181,5 @@ type kvEntry struct {
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -11,7 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -27,9 +29,16 @@ type passer struct {
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
|
||||
failedProvisions map[string]bool
|
||||
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
|
||||
missing, missingSaid map[string]bool
|
||||
// unplaced are the nodes whose unplaced finding this process wrote.
|
||||
unplaced map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
@@ -40,6 +49,25 @@ type passer struct {
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
|
||||
// provider is down, which holds no want): said once per words, and once more when it is reached again.
|
||||
unreached map[string]string
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
|
||||
// (clearStaleCalls): the assigner makes no such call now (issue 774).
|
||||
callsCleared bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// convertedAll says every node in the form before was converted to its node and module keys (convertPlacement).
|
||||
convertedAll bool
|
||||
// saidWaits is the last said of the seat's wait for its states (sayWaits).
|
||||
saidWaits string
|
||||
intended map[string]map[string]map[string]bool
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -50,26 +78,27 @@ func (p *passer) clock() time.Time {
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
p.placementSeat(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
@@ -79,7 +108,7 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
for name, mod := range trunk {
|
||||
@@ -98,6 +127,14 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
provisionsOf := p.provisionsOf()
|
||||
missing := map[string]bool{}
|
||||
defer func() { p.provisionsMissing(ctx, missing) }()
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
@@ -114,15 +151,21 @@ func (p *passer) wants(ctx context.Context) {
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
key, err := delivery.PartGrantKey(name, fp)
|
||||
return err == nil && p.w.has(grants, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
@@ -134,13 +177,13 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.PartGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
@@ -153,8 +196,17 @@ func (p *passer) wants(ctx context.Context) {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
for _, node := range m.Nodes {
|
||||
for _, prov := range m.MissingProvisions(node) {
|
||||
missing[node+"."+name+"."+prov] = true
|
||||
}
|
||||
}
|
||||
p.yeses(ctx, name, m, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
p.sayUnreached(name, m.Nodes, out)
|
||||
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
@@ -210,7 +262,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
reason = h.Reason()
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
@@ -229,6 +281,80 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
}
|
||||
|
||||
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
|
||||
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
|
||||
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
|
||||
if p.unreached == nil {
|
||||
p.unreached = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
key := node + "." + module
|
||||
why, last := out.Unreached[node], p.unreached[key]
|
||||
switch {
|
||||
case why != "" && why != last:
|
||||
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
|
||||
p.unreached[key] = why
|
||||
case why == "" && last != "":
|
||||
p.logf("%s on %s is reached: its providers run healthy", module, node)
|
||||
delete(p.unreached, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
|
||||
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
|
||||
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
|
||||
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
|
||||
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
|
||||
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
|
||||
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
|
||||
if p.nodeFailed == nil {
|
||||
p.nodeFailed = map[string]string{}
|
||||
}
|
||||
look := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
look[node] = true
|
||||
}
|
||||
for key := range p.nodeFailed {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
if first {
|
||||
for node := range known.Online {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(look)) {
|
||||
key := node + "." + module
|
||||
f, failing := out.NodeFailed[node]
|
||||
last, said := p.nodeFailed[key]
|
||||
switch {
|
||||
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
|
||||
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
|
||||
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
|
||||
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
|
||||
case !failing && (said || first):
|
||||
if said {
|
||||
p.logf("%s on %s no longer fails its health check alone", module, node)
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
|
||||
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
delete(p.nodeFailed, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
@@ -242,6 +368,48 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
@@ -258,7 +426,9 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
@@ -278,73 +448,30 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||
func (p *passer) compositions(ctx context.Context) {
|
||||
if p.sdk == "" {
|
||||
p.sdk = sdkversion.Version()
|
||||
}
|
||||
written := p.w.read(delivery.AssignerCompositionsBucket)
|
||||
trunk := p.w.trunk()
|
||||
var mesh *assign.Mesh
|
||||
seen := map[string]bool{}
|
||||
if p.asks != nil {
|
||||
defer p.asks.sweep(seen)
|
||||
}
|
||||
for key, value := range p.w.read(delivery.CandidatesBucket) {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err)
|
||||
continue
|
||||
}
|
||||
out, err := delivery.CompositionKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s names no composition key: %v", key, err)
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk)
|
||||
var v delivery.CompositionVerdict
|
||||
if waits != "" {
|
||||
v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits}
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
if mesh == nil {
|
||||
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
|
||||
mesh = &m
|
||||
}
|
||||
if ch.SDKHead != "" && p.asks != nil {
|
||||
// A group with a mesh-sdk member is composed by the composer built at its head (ADR 0482 clauses 3 to 5).
|
||||
v = p.composeAtHead(ctx, *mesh, ch, seen, began)
|
||||
} else {
|
||||
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||
}
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||
before.Took, before.V = v.Took, 0
|
||||
if delivery.SameCompositionVerdict(before, v) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) })
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -355,6 +482,11 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
p.logf("%s %s is not written: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
// A value the bucket already holds is not put again: every put is a change its watchers hear, and a want put unchanged
|
||||
// made its node compose again (issue 774, 2026-10-31: novox composed every 8 s, two asks of the vault each time).
|
||||
if held := p.w.value(bucket, key); held != nil && bytes.Equal(held, value) {
|
||||
return
|
||||
}
|
||||
if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be written: %v", bucket, key, err)
|
||||
return
|
||||
@@ -434,6 +566,14 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||
if !on {
|
||||
return true
|
||||
}
|
||||
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
|
||||
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
|
||||
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
|
||||
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||
@@ -675,3 +815,272 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if m.Self == nil || !m.Self(node) {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes := ""
|
||||
if had && was.FP == w.FP {
|
||||
yes = was.Yes
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
if m.Allowed != nil && m.Allowed(node, h.Waits) {
|
||||
yes = h.Waits
|
||||
} else {
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes != w.Yes || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Composed, w.Self = yes, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chosenOf is a person's choice for module, nil when none.
|
||||
func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy {
|
||||
if c, ok := chosen[module]; ok {
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
|
||||
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
|
||||
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
|
||||
// needs-operator), deleted once it is no longer failed.
|
||||
func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision {
|
||||
if p.kvs[provisions.ProvisionsBucket] == nil {
|
||||
return func(string) func(string) map[string]provisions.Provision { return nil }
|
||||
}
|
||||
by := map[string]map[string]map[string]provisions.Provision{} // module, node, key
|
||||
failed := map[string]provisions.Provision{}
|
||||
for key, raw := range p.w.read(provisions.ProvisionsBucket) {
|
||||
pr, err := provisions.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
node, module := pr.Consumer.Node, pr.Consumer.Module
|
||||
if by[module] == nil {
|
||||
by[module] = map[string]map[string]provisions.Provision{}
|
||||
}
|
||||
if by[module][node] == nil {
|
||||
by[module][node] = map[string]provisions.Provision{}
|
||||
}
|
||||
by[module][node][key] = pr
|
||||
if pr.State == provisions.Failed {
|
||||
failed[key] = pr
|
||||
}
|
||||
}
|
||||
p.provisionFindings(failed)
|
||||
return func(module string) func(string) map[string]provisions.Provision {
|
||||
return func(node string) map[string]provisions.Provision { return by[module][node] }
|
||||
}
|
||||
}
|
||||
|
||||
// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed.
|
||||
func (p *passer) provisionFindings(failed map[string]provisions.Provision) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now
|
||||
if p.now != nil {
|
||||
now = p.now
|
||||
}
|
||||
for key, pr := range failed {
|
||||
if p.failedProvisions[key] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionFailedFinding(key, pr, now())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionFailedKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the failed provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.failedProvisions == nil {
|
||||
p.failedProvisions = map[string]bool{}
|
||||
}
|
||||
p.failedProvisions[key] = true
|
||||
}
|
||||
for key := range p.failedProvisions {
|
||||
if _, still := failed[key]; still {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil {
|
||||
p.logf("the failed provision %s is not cleared: %v", key, err)
|
||||
continue
|
||||
}
|
||||
delete(p.failedProvisions, key)
|
||||
}
|
||||
}
|
||||
|
||||
// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
|
||||
// when its manifest names none (ADR 0564 clause 13 (d)).
|
||||
var MissingBound = delivery.DefaultDeliveryBound
|
||||
|
||||
// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
|
||||
func missingPulse(key string) string { return "provision-missing." + key }
|
||||
|
||||
// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
|
||||
// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
|
||||
// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
|
||||
// the watch, is the pass that writes it: no timer.
|
||||
func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
|
||||
pulses := p.kvs[delivery.PlacementPulsesBucket]
|
||||
if p.missing == nil {
|
||||
p.missing = map[string]bool{}
|
||||
}
|
||||
for key := range missing {
|
||||
switch {
|
||||
case !p.missing[key]:
|
||||
p.missing[key] = true
|
||||
if pulses != nil {
|
||||
p.pulse(ctx, missingPulse(key), MissingBound)
|
||||
}
|
||||
case pulses == nil || !p.pulsing(missingPulse(key)):
|
||||
if p.missingSaid[key] || p.findings.Put == nil {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionMissingKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the missing provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.missingSaid == nil {
|
||||
p.missingSaid = map[string]bool{}
|
||||
}
|
||||
p.missingSaid[key] = true
|
||||
}
|
||||
}
|
||||
for key := range p.missing {
|
||||
if missing[key] {
|
||||
continue
|
||||
}
|
||||
delete(p.missing, key)
|
||||
if pulses != nil && p.pulsing(missingPulse(key)) {
|
||||
if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil)
|
||||
}
|
||||
}
|
||||
if p.missingSaid[key] && p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
|
||||
p.logf("the missing provision %s is not cleared: %v", key, err)
|
||||
p.missing[key] = true
|
||||
continue
|
||||
}
|
||||
delete(p.missingSaid, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unplacedFindings writes, from the lease write, the finding `machine.<node>.unplaced` for each node that runs but no
|
||||
// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed.
|
||||
func (p *passer) unplacedFindings() {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := []string(p.w.unplacedNodes())
|
||||
is := map[string]bool{}
|
||||
for _, node := range now {
|
||||
is[node] = true
|
||||
if p.unplaced[node] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.UnplacedFinding(node, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.UnplacedKey(node), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the unplaced node %s is not said: %v", node, err)
|
||||
continue
|
||||
}
|
||||
if p.unplaced == nil {
|
||||
p.unplaced = map[string]bool{}
|
||||
}
|
||||
p.unplaced[node] = true
|
||||
}
|
||||
for node := range p.unplaced {
|
||||
if is[node] {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil {
|
||||
delete(p.unplaced, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per
|
||||
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
|
||||
// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no
|
||||
// timer writes it.
|
||||
func (p *passer) seatHolders(ctx context.Context) {
|
||||
kv := p.kvs[delivery.SeatHoldersBucket]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
placements := p.w.placements()
|
||||
if len(placements) == 0 {
|
||||
return // no placement read yet: nothing is known of who holds what, and nothing is deleted
|
||||
}
|
||||
want := delivery.SeatHoldersOf(placements)
|
||||
have := p.w.read(delivery.SeatHoldersBucket)
|
||||
for key, sh := range want {
|
||||
value, err := json.Marshal(sh)
|
||||
if err != nil || string(have[key]) == string(value) {
|
||||
continue
|
||||
}
|
||||
p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil })
|
||||
}
|
||||
for key := range have {
|
||||
if _, ok := want[key]; ok {
|
||||
continue
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil {
|
||||
p.logf("the seat holders %s are not deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.SeatHoldersBucket, key, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
|
||||
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
|
||||
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
|
||||
return func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) {
|
||||
if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -15,7 +16,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -24,9 +24,16 @@ import (
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
puts int // how many puts reached the bucket
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
m.puts++
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
@@ -42,6 +49,34 @@ func newPasser(kvs buckets, at time.Time) *passer {
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
|
||||
// (research 072 row 37).
|
||||
func placeOn(t *testing.T, w *world, node string, modules ...string) {
|
||||
t.Helper()
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
|
||||
func composes(t *testing.T, w *world, node, digest string) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
@@ -135,6 +170,9 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
if s[state] == nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
@@ -198,7 +236,7 @@ func putOfTheBus(t *testing.T) {
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
placeOn(t, p.w, "novox", "nats")
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
@@ -254,12 +292,10 @@ func putOfTheBus(t *testing.T) {
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||
composes(t, p.w, "novox", digest)
|
||||
gkey, _ := delivery.PartGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
world(delivery.PartGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
@@ -284,7 +320,7 @@ func putOfTheBus(t *testing.T) {
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
@@ -342,6 +378,9 @@ func putOfTheBus(t *testing.T) {
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
@@ -357,30 +396,66 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
@@ -399,6 +474,7 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
@@ -406,19 +482,78 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
|
||||
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
|
||||
for i, c := range []struct {
|
||||
failed map[string]assign.NodeFailure
|
||||
state string // the condition said in this pass, "" for none
|
||||
}{
|
||||
{failing, asks.StateRaised},
|
||||
{failing, ""},
|
||||
{nil, asks.StateCleared},
|
||||
{nil, ""},
|
||||
} {
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
|
||||
switch {
|
||||
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
|
||||
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
|
||||
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
|
||||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
|
||||
t.Errorf("node failed, raised %+v", conds[0])
|
||||
}
|
||||
}
|
||||
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
|
||||
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
|
||||
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
|
||||
}
|
||||
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
|
||||
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
known assign.Nodes
|
||||
keys []string
|
||||
}{
|
||||
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
} {
|
||||
conds = nil
|
||||
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
|
||||
fresh.logf = func(string, ...any) {}
|
||||
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
|
||||
var keys []string
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
keys = append(keys, cond.Key)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
|
||||
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -449,14 +584,14 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
placeOn(t, p.w, "shanks", "systemd-resolved")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
@@ -520,15 +655,13 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
noDigit bool // the node's own composition is not read
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
|
||||
{"the trunk is not read at all", nil, nil, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
@@ -555,21 +688,9 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
|
||||
if !c.noDigit {
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
composes(t, p.w, "novox", digNew)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
@@ -665,51 +786,145 @@ func mustKey(key string, err error) string {
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
}
|
||||
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
|
||||
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
|
||||
if got := p.w.assigned(); len(got) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v", got)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "c", "a")
|
||||
placeOn(t, p.w, "n2", "d")
|
||||
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
|
||||
t.Errorf("the placement alone: assigned %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
|
||||
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
|
||||
// 8 s, and a want put unchanged each pass made the node compose again each time.
|
||||
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
|
||||
asked := asksKV.values[akey]
|
||||
p.wants(context.Background())
|
||||
if !bytes.Equal(asksKV.values[akey], asked) {
|
||||
t.Fatalf("the record ask the node's have waits on was taken away or made again: %q", asksKV.values[akey])
|
||||
}
|
||||
wantPuts, askPuts := wantsKV.puts, asksKV.puts
|
||||
p.wants(context.Background())
|
||||
if wantsKV.puts != wantPuts || asksKV.puts != askPuts {
|
||||
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
|
||||
}
|
||||
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
|
||||
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
|
||||
if wantsKV.puts != wantPuts {
|
||||
t.Fatalf("a want put unchanged reached the bucket")
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
}
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
|
||||
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
|
||||
// cleared once, and said again at a later pass while the findings cannot be read.
|
||||
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
|
||||
var said []string
|
||||
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
|
||||
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
|
||||
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := 1
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
|
||||
}
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
if len(conds) != 2 || conds[0].State != asks.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -46,6 +47,8 @@ type putActs struct {
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
@@ -53,7 +56,7 @@ type putActs struct {
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
@@ -94,7 +97,7 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned, _ := p.w.assigned()
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
@@ -183,7 +186,10 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
if policy == "" {
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
if c, ok := chosen[module]; ok {
|
||||
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
|
||||
} else if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
@@ -213,17 +219,16 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
@@ -382,8 +387,9 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and
|
||||
// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
@@ -393,10 +399,10 @@ func (a *putActs) warrant(id, ask string) {
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,748 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
|
||||
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
|
||||
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
|
||||
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
|
||||
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
|
||||
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
|
||||
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
|
||||
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
|
||||
|
||||
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
|
||||
type publisher func(event string, body any, id string) error
|
||||
|
||||
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
|
||||
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", delivery.AssignerModule)
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
|
||||
// the controller: placement is the one source (ADR 0567 §8, issue 774).
|
||||
type seatActs struct {
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
|
||||
}
|
||||
|
||||
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
|
||||
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
}
|
||||
ask, _, _ := strings.Cut(warrant, "@")
|
||||
go func() {
|
||||
if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
} else {
|
||||
a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
|
||||
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
|
||||
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
|
||||
func (p *passer) clearStaleCalls() {
|
||||
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
|
||||
return
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.callsCleared = true
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "module.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
|
||||
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
|
||||
if node, module, two := strings.Cut(pair, "."); two {
|
||||
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
|
||||
p.callsCleared = false
|
||||
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
func (p *passer) sayWaits(unbound []string) {
|
||||
c := assign.SeatWaitsCondition(unbound)
|
||||
state := string(c.State) + c.Summary
|
||||
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
|
||||
return
|
||||
}
|
||||
if c.State == asks.StateCleared && p.saidWaits == "" {
|
||||
return // nothing was raised by this process
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key], p.saidWaits = true, state
|
||||
}
|
||||
|
||||
// seatPlacements is every node's placement and the revision of its node key (its legacy key's for a node not yet
|
||||
// converted), as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
|
||||
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
|
||||
return p.w.placement.AllRevisions()
|
||||
}
|
||||
|
||||
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
|
||||
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
|
||||
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
|
||||
var newest time.Time
|
||||
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
out[node], revs[node] = c, meta.revision
|
||||
if meta.at.After(newest) {
|
||||
newest = meta.at
|
||||
}
|
||||
}
|
||||
return out, revs, newest
|
||||
}
|
||||
|
||||
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
|
||||
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[id]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placementSeat takes every open proposal one step further.
|
||||
func (p *passer) placementSeat(ctx context.Context) {
|
||||
for _, b := range seatBuckets {
|
||||
if p.kvs[b] == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.saidProposed == nil {
|
||||
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
current, revs := p.seatPlacements()
|
||||
ids := make([]string, 0)
|
||||
raws := p.w.read(delivery.PlacementProposalsBucket)
|
||||
for id := range raws {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
prop, err := delivery.DecodePlacementProposal(raws[id])
|
||||
if err != nil || prop.ID != id {
|
||||
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
|
||||
continue
|
||||
}
|
||||
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
|
||||
}
|
||||
}
|
||||
|
||||
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
|
||||
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
|
||||
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
|
||||
kv := p.kvs[delivery.PlacementPulsesBucket]
|
||||
_ = kv.Delete(ctx, key)
|
||||
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
|
||||
if err != nil {
|
||||
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
|
||||
}
|
||||
|
||||
// pulsing says whether a bound's key stands.
|
||||
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
|
||||
|
||||
// deletePulses deletes every bound of a proposal.
|
||||
func (p *passer) deletePulses(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementPulsesBucket) {
|
||||
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
|
||||
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
|
||||
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
|
||||
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
|
||||
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
|
||||
return prop, true
|
||||
}
|
||||
|
||||
// stepProposal takes one proposal one step further.
|
||||
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
|
||||
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
|
||||
p.saidProposed[prop.ID] = true
|
||||
}
|
||||
if prop.Withdrawn != "" {
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
if prop.Seed || prop.HandBack {
|
||||
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
|
||||
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
|
||||
return
|
||||
}
|
||||
if prop.Warrant != nil {
|
||||
p.addNode(ctx, prop, revs)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
return
|
||||
case !p.pulsing(askPulse) && !again:
|
||||
p.endProposal(ctx, prop, "its ask expired unanswered")
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
}
|
||||
cands, candRevs, written := p.candidatesOf(prop.ID)
|
||||
if len(prop.Must) == 0 {
|
||||
p.mustAnswer(ctx, prop, rev, current, cands)
|
||||
return
|
||||
}
|
||||
for _, node := range prop.Must {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // a candidate just written is not heard yet: its watch wakes the next pass
|
||||
}
|
||||
}
|
||||
if written.After(prop.Written) {
|
||||
prop.Written = written
|
||||
}
|
||||
pulsing := map[string]bool{}
|
||||
for _, node := range prop.Must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
pulsing[node] = p.pulsing(k)
|
||||
}
|
||||
}
|
||||
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
|
||||
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
|
||||
return
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case delivery.VerdictFail:
|
||||
prop.State, prop.Ask = delivery.ProposalFailed, ""
|
||||
case delivery.VerdictWaits:
|
||||
prop.State, prop.Ask = delivery.ProposalAnswering, ""
|
||||
case delivery.VerdictPass:
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
}
|
||||
if again {
|
||||
prop.Words = "judged and asked again (again)"
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.writeProposal(ctx, prop, rev) {
|
||||
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
|
||||
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
|
||||
}
|
||||
}
|
||||
|
||||
// sameVerdict says two verdicts say the same, their time aside.
|
||||
func sameVerdict(a, b delivery.PlacementVerdict) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
|
||||
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
|
||||
for node, base := range prop.Base {
|
||||
if revs[node] != base {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
|
||||
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
|
||||
key := fmt.Sprint(prop.ID, prop.Base)
|
||||
if got, ok := p.intended[key]; ok {
|
||||
return got
|
||||
}
|
||||
got, _ := p.compute(prop, current, cands)
|
||||
p.intended[key] = got
|
||||
return got
|
||||
}
|
||||
|
||||
// compute is what the proposal changes on each node by its placement, and the nodes that answer it: every node with a
|
||||
// placement or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
|
||||
// change in its answer.
|
||||
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string) {
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
if c, ok := cands[node]; ok && !c.Reached {
|
||||
laid[node] = c.Placement
|
||||
}
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for n := range current {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range p.w.records() {
|
||||
nodes[n] = true
|
||||
}
|
||||
return judge.Intended(current, laid, slices.Sorted(maps.Keys(nodes)))
|
||||
}
|
||||
|
||||
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
|
||||
// change, and writes them into the proposal.
|
||||
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
|
||||
for _, node := range prop.Nodes() {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // the verbs write the candidates before the proposal: wait to hear them
|
||||
}
|
||||
}
|
||||
intended, must := p.compute(prop, current, cands)
|
||||
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
|
||||
for _, node := range must {
|
||||
if _, ok := cands[node]; ok {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
|
||||
if pl, ok := current[node]; ok {
|
||||
c.Placement = &pl
|
||||
}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
for _, node := range must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
p.pulse(ctx, k, judge.AnswerBound)
|
||||
}
|
||||
}
|
||||
prop.Must = must
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
|
||||
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
|
||||
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
for _, node := range again.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
|
||||
// nodes with a record.
|
||||
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
|
||||
held := map[string]bool{}
|
||||
for key := range p.w.read(delivery.ModulesBucket) {
|
||||
if name, err := delivery.ParseModulesKey(key); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
records := p.w.records()
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, and placement-changed said.
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
}
|
||||
cands, _, _ := p.candidatesOf(prop.ID)
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
laid[node] = cands[node].Placement
|
||||
}
|
||||
writes, err := assign.Writes(prop, laid)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
var changes []delivery.PlacementChange
|
||||
writtenLaid := map[string]*delivery.Placement{}
|
||||
after := maps.Clone(current)
|
||||
afterRevs := maps.Clone(revs)
|
||||
for _, w := range writes {
|
||||
rev2, err := p.writeNode(ctx, w.Node, w.Value)
|
||||
if err != nil {
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
before := current[w.Node]
|
||||
now := delivery.Placement{Node: w.Node}
|
||||
if w.Value != nil {
|
||||
now = *w.Value
|
||||
after[w.Node] = *w.Value
|
||||
} else {
|
||||
delete(after, w.Node)
|
||||
}
|
||||
afterRevs[w.Node] = rev2
|
||||
c := delivery.ChangeOf(before, now)
|
||||
c.Before, c.After = w.Base, rev2
|
||||
changes = append(changes, c)
|
||||
writtenLaid[w.Node] = w.Value
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes)
|
||||
}
|
||||
|
||||
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
|
||||
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
|
||||
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
|
||||
// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
|
||||
func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) {
|
||||
current, _ := p.seatPlacements()
|
||||
if moved(prop, revs) {
|
||||
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
|
||||
return
|
||||
}
|
||||
laid, findings := delivery.LayChanges(current, prop.Changes)
|
||||
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
|
||||
node := prop.Changes[0].Node
|
||||
if prop.Changes[0].Op == delivery.OpRemoveNode {
|
||||
p.removeNode(ctx, prop, current, findings)
|
||||
return
|
||||
}
|
||||
value, ok := laid[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, &value)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(delivery.Placement{Node: node}, value)
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
|
||||
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant.
|
||||
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
|
||||
func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) {
|
||||
node := prop.Changes[0].Node
|
||||
before, ok := current[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, nil)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
|
||||
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
|
||||
at = t
|
||||
}
|
||||
}
|
||||
delivery.SortChanges(changes)
|
||||
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
|
||||
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
|
||||
p.w.put(delivery.AssignerAsksBucket, akey, nil)
|
||||
}
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
// deleteCandidates deletes every candidate of a proposal.
|
||||
func (p *passer) deleteCandidates(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
|
||||
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
|
||||
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
|
||||
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
|
||||
raw, err := delivery.EncodePlacementProposal(prop)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s is not written: %v", prop.ID, err)
|
||||
return false
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementProposalsBucket]
|
||||
var at uint64
|
||||
if rev == 0 {
|
||||
at, err = kv.Create(ctx, prop.ID, raw)
|
||||
} else {
|
||||
at, err = kv.Update(ctx, prop.ID, raw, rev)
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
|
||||
return true
|
||||
}
|
||||
|
||||
// say says one of the seat's events, said in the journal when the bus refuses it.
|
||||
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
|
||||
if p.publish == nil {
|
||||
return
|
||||
}
|
||||
if err := p.publish(event, body, id); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// writeNode writes one node's placement after (nil: none) as its keys (mesh-sdk delivery.PlacementKeys.Writes; research
|
||||
// 072 row 37): the module keys that change, each carrying the value it replaces, then the node key, the commit, then the
|
||||
// module keys no longer placed. It answers the node key's revision after (0: deleted). A write that fails up to the
|
||||
// commit ends it with nothing changed for any reader, the module keys written before it read at their previous values;
|
||||
// a delete after the commit that fails leaves a module key the node key does not name, which places nothing and is
|
||||
// deleted at the node's next write. Each key written is taken into the world at once, so the next write of the pass
|
||||
// reads it.
|
||||
func (p *passer) writeNode(ctx context.Context, node string, after *delivery.Placement) (uint64, error) {
|
||||
writes, err := p.w.placement.Writes(node, after)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementBucket]
|
||||
committed, rev := false, uint64(0)
|
||||
for _, w := range writes {
|
||||
got, err := writeKey(ctx, kv, w)
|
||||
if err != nil {
|
||||
if committed {
|
||||
p.logf("placement %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
|
||||
continue
|
||||
}
|
||||
return 0, fmt.Errorf("placement %s at revision %d: %w", w.Key, w.Base, err)
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementBucket, w.Key, w.Value, got, p.clock())
|
||||
if w.Commit {
|
||||
committed, rev = true, got
|
||||
}
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// writeKey makes one write of a placement key by compare-and-set and answers the key's revision after (0: deleted).
|
||||
func writeKey(ctx context.Context, kv jetstream.KeyValue, w delivery.PlacementKeyWrite) (uint64, error) {
|
||||
switch {
|
||||
case w.Value == nil:
|
||||
return 0, kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
|
||||
case w.Base == 0:
|
||||
return kv.Create(ctx, w.Key, w.Value)
|
||||
default:
|
||||
return kv.Update(ctx, w.Key, w.Value, w.Base)
|
||||
}
|
||||
}
|
||||
|
||||
// convertPlacement converts every node the state holds only in the form before, `placement.<node>`, into its node key
|
||||
// and module keys (research 072 row 37), once: each node converted is said, and a node whose conversion fails is said
|
||||
// and tried again at the next pass. The legacy key is left for the readers not yet delivered; a node key wins over it
|
||||
// for every reader that reads both, and it is deleted with the node key when the node's placement is.
|
||||
func (p *passer) convertPlacement(ctx context.Context) {
|
||||
if p.convertedAll {
|
||||
return
|
||||
}
|
||||
migrations, err := p.w.placement.Migrations()
|
||||
if err != nil {
|
||||
p.logf("placement: some nodes in the form before cannot be converted, and stay read from it: %v", err)
|
||||
}
|
||||
failed := false
|
||||
for _, node := range slices.Sorted(maps.Keys(migrations)) {
|
||||
before, rev, _ := p.w.placement.Of(node)
|
||||
got, err := p.writeNode(ctx, node, &before)
|
||||
if err != nil {
|
||||
failed = true
|
||||
p.logf("placement: %s is not converted from placement.%s (revision %d) to its node and module keys, and is tried again at the next pass: %v", node, node, rev, err)
|
||||
continue
|
||||
}
|
||||
p.logf("placement: %s converted from placement.%s (revision %d) to %s.node (revision %d) and %d module keys (research 072 row 37); placement.%s stays for readers not yet delivered",
|
||||
node, node, rev, node, got, len(before.Modules), node)
|
||||
}
|
||||
p.convertedAll = !failed && err == nil
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
|
||||
|
||||
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
|
||||
type seatStore interface {
|
||||
store
|
||||
del(state, key string) error
|
||||
history(state, key string) ([]stdio.StateRevision, error)
|
||||
}
|
||||
|
||||
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
held := delivery.NewPlacementKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
out, revs := held.AllRevisions()
|
||||
if len(bad) > 0 && len(out) == 0 {
|
||||
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
raw = []byte(c)
|
||||
case nil:
|
||||
return nil, errors.New("propose names its changes")
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// doPropose is the verb propose (§2).
|
||||
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
changes, err := changesOf(args["changes"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warrant, err := warrantOf(args["warrant"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
|
||||
if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
|
||||
func warrantOf(v any) (*delivery.AddNodeWarrant, error) {
|
||||
var raw []byte
|
||||
switch w := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
if strings.TrimSpace(w) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw = []byte(w)
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out delivery.AddNodeWarrant
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
|
||||
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
|
||||
// as proven, which the refusal names.
|
||||
func moduleOfCaller(by string) string {
|
||||
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
|
||||
return delivery.EnrolmentModule
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
|
||||
}
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
}
|
||||
|
||||
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
|
||||
// when its verdict passes.
|
||||
func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
History int `json:"history-per-key"`
|
||||
Changes []historyEntry `json:"changes,omitempty"`
|
||||
Older string `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
if arg(args, "seats") == "true" {
|
||||
return s.seats(arg(args, "node"))
|
||||
}
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
}
|
||||
nodes := out.Nodes
|
||||
if node != "" {
|
||||
nodes = []string{node}
|
||||
} else if module == "" {
|
||||
nodes = nil
|
||||
for n := range all {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
revisions, err := s.nodeHistory(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var before sketch
|
||||
for _, r := range revisions {
|
||||
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
|
||||
after := sketch{}
|
||||
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
|
||||
after = sketchOf(r.Key, r.Value)
|
||||
} else {
|
||||
e.Deleted = true
|
||||
}
|
||||
e.Change = after.changeFrom(n, before)
|
||||
e.Change.Before, e.Change.After = 0, r.Revision
|
||||
before = after
|
||||
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := []openProposal{}
|
||||
for _, id := range ids {
|
||||
if want := arg(args, "id"); want != "" && want != id {
|
||||
continue
|
||||
}
|
||||
p, err := s.proposal(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if arg(args, "id") != "" && len(out) == 0 {
|
||||
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
|
||||
}
|
||||
return map[string]any{"proposals": out}, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
|
||||
// only from a tool of that name), with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
keys, err := s.st.keys(delivery.SeatHoldersState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []delivery.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var sh delivery.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
|
||||
continue
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return map[string]any{"seats": out, "holds": len(keys)}, nil
|
||||
}
|
||||
|
||||
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
|
||||
type keyedRevision struct {
|
||||
stdio.StateRevision
|
||||
Key string
|
||||
}
|
||||
|
||||
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
|
||||
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
|
||||
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
|
||||
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
|
||||
legacy, err := delivery.LegacyPlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodeKey, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []keyedRevision
|
||||
for _, key := range []string{legacy, nodeKey} {
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, r := range revisions {
|
||||
out = append(out, keyedRevision{StateRevision: r, Key: key})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
|
||||
// and holdings, which change every module (delivery.ChangeOf).
|
||||
type sketch struct {
|
||||
modules []delivery.PlacementRef
|
||||
nodeWide string
|
||||
}
|
||||
|
||||
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
|
||||
func sketchOf(key string, value []byte) sketch {
|
||||
_, _, kind, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
|
||||
raw, _ := json.Marshal([]any{pins, holds})
|
||||
return string(raw)
|
||||
}
|
||||
if kind == delivery.PlacementNodeKind {
|
||||
n, err := delivery.DecodePlacementNode(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
|
||||
for _, m := range p.Modules {
|
||||
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
|
||||
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
|
||||
c := delivery.PlacementChange{Node: node}
|
||||
was := map[string]string{}
|
||||
for _, r := range before.modules {
|
||||
was[r.Module] = r.Digest
|
||||
}
|
||||
now := map[string]bool{}
|
||||
nodeWide := before.nodeWide != s.nodeWide && len(before.modules) > 0
|
||||
for _, r := range s.modules {
|
||||
now[r.Module] = true
|
||||
d, ok := was[r.Module]
|
||||
switch {
|
||||
case !ok:
|
||||
c.Added = append(c.Added, r.Module)
|
||||
case nodeWide || d != r.Digest:
|
||||
c.Changed = append(c.Changed, r.Module)
|
||||
}
|
||||
}
|
||||
for _, r := range before.modules {
|
||||
if !now[r.Module] {
|
||||
c.Removed = append(c.Removed, r.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(c.Added)
|
||||
sort.Strings(c.Removed)
|
||||
sort.Strings(c.Changed)
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
func (s memStore) del(state, key string) error {
|
||||
delete(s[state].values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
v, ok := s[state].values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
st := memStore{}
|
||||
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
for _, m := range []string{"web", "shell"} {
|
||||
st[modulesState].values[m] = []byte("{}")
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
|
||||
t.Errorf("marked = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
|
||||
t.Error("an unknown proposal was withdrawn")
|
||||
}
|
||||
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
t.Error("an unknown change was taken")
|
||||
}
|
||||
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
|
||||
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
move := got.(proposed).Proposal
|
||||
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
|
||||
t.Fatalf("propose = %+v", move)
|
||||
}
|
||||
|
||||
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
|
||||
reads := map[string]bool{}
|
||||
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
|
||||
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
|
||||
t.Errorf("placement of web = %+v, %v", pa, err)
|
||||
} else {
|
||||
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
|
||||
}
|
||||
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
|
||||
t.Error("a node with no key was answered")
|
||||
}
|
||||
ps, err := s.doProposals(map[string]any{"id": move.ID})
|
||||
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
|
||||
t.Errorf("proposals = %+v, %v", ps, err)
|
||||
} else {
|
||||
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
|
||||
}
|
||||
raw0, err := readManifest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range raw0.State {
|
||||
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
|
||||
t.Errorf("no verb reads the seat's state %s", state.Name)
|
||||
}
|
||||
}
|
||||
for _, stream := range raw0.Streams {
|
||||
if !reads["stream "+stream.Name] {
|
||||
t.Errorf("no verb reads the seat's stream %s", stream.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold: %v", err)
|
||||
}
|
||||
// 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with
|
||||
// the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused.
|
||||
addNode := func(by, warrant string) error {
|
||||
_, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace",
|
||||
"decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by)
|
||||
return err
|
||||
}
|
||||
good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`
|
||||
for _, c := range []struct{ by, warrant, want string }{
|
||||
{"novox.mesh-grants", good, "mesh-enrolment's alone"},
|
||||
{"person.mesh-enrolment", good, "mesh-enrolment's alone"},
|
||||
{"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
{"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"},
|
||||
{"novox.mesh-enrolment", "", "carries no warrant"},
|
||||
} {
|
||||
if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want)
|
||||
}
|
||||
}
|
||||
if err := addNode("novox.mesh-enrolment", good); err != nil {
|
||||
t.Errorf("an add-node from mesh-enrolment: %v", err)
|
||||
}
|
||||
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
|
||||
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
|
||||
// of a node running modules, or with another node's warrant, it is refused.
|
||||
joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}})
|
||||
st[delivery.PlacementState].values["placement.joiner"] = joiner
|
||||
removeNode := func(node, warrant string) (any, error) {
|
||||
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
|
||||
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
|
||||
}
|
||||
for _, c := range []struct{ node, warrant, want string }{
|
||||
{"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"},
|
||||
{"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
} {
|
||||
if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want)
|
||||
}
|
||||
}
|
||||
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
|
||||
t.Errorf("a remove-node from mesh-enrolment: %v", err)
|
||||
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
|
||||
t.Errorf("the remove-node proposed %+v", rm)
|
||||
}
|
||||
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
|
||||
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey, _ := delivery.LegacyPlacementKey("ace")
|
||||
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
|
||||
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
|
||||
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
hk, _ := delivery.HaveNodeKey(node)
|
||||
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
|
||||
}
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Errorf("ace: %+v %v", d, ok)
|
||||
}
|
||||
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
|
||||
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
}
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
p.unplacedFindings()
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey2, _ := delivery.LegacyPlacementKey("g14")
|
||||
w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now())
|
||||
p.unplacedFindings()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("placed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
|
||||
writes := 0
|
||||
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
|
||||
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}}
|
||||
place := func(node string, holds ...delivery.Holding) {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := delivery.LegacyPlacementKey(node)
|
||||
p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now())
|
||||
}
|
||||
place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
ctx := context.Background()
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
|
||||
t.Fatalf("%d writes: %v", writes, kv.values)
|
||||
}
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 {
|
||||
t.Errorf("an unchanged placement wrote %d keys", writes-3)
|
||||
}
|
||||
place("g14") // g14 holds nothing now
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
|
||||
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
|
||||
}
|
||||
st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}}
|
||||
out, err := newSeatTools(st, nil).seats("ace")
|
||||
if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 {
|
||||
t.Errorf("seats on ace = %+v, %v", out, err)
|
||||
}
|
||||
})
|
||||
t.Run("research 072 row 37: the form before is converted once, a node is written as its keys, and history reads both", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var logs []string
|
||||
p := &passer{w: newWorld(), kvs: buckets{delivery.PlacementBucket: kv}, now: time.Now,
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
put := func(key string, value []byte, rev uint64) {
|
||||
p.w.putEntry(delivery.PlacementBucket, key, value, rev, time.Now())
|
||||
}
|
||||
was := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
raw, _ := delivery.EncodePlacement(was)
|
||||
put("placement.anchor", raw, kv.write("placement.anchor", raw))
|
||||
p.convertPlacement(ctx)
|
||||
if strings.Join(kv.log, " ") != "create anchor.web create anchor.node" || !p.convertedAll ||
|
||||
!strings.Contains(strings.Join(logs, "\n"), "anchor converted from placement.anchor") {
|
||||
t.Fatalf("the conversion wrote %v, said %v", kv.log, logs)
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
if len(kv.log) != 2 {
|
||||
t.Errorf("a converted node is converted again: %v", kv.log)
|
||||
}
|
||||
// A write: the module keys, the node key (its commit), the mirror, then the module key no longer placed.
|
||||
after := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2222}}}}
|
||||
kv.log = nil
|
||||
rev, err := p.writeNode(ctx, "anchor", &after)
|
||||
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node update placement.anchor delete anchor.web" || rev != kv.revs["anchor.node"] {
|
||||
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
|
||||
}
|
||||
if got, _ := p.seatPlacements(); !delivery.SamePlacement(got["anchor"], after) {
|
||||
t.Errorf("the write is not read back at once: %+v", got)
|
||||
}
|
||||
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
|
||||
kv.refuse, kv.log = "anchor.node", nil
|
||||
moved := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2223}}}}
|
||||
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
|
||||
t.Fatalf("a refused commit = %v, %v", kv.log, err)
|
||||
}
|
||||
fresh, err := delivery.PlacementsOf(kv.values)
|
||||
if err != nil || !delivery.SamePlacement(fresh["anchor"], after) {
|
||||
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
|
||||
}
|
||||
kv.refuse = ""
|
||||
// The verb's history reads the form before, then the node key.
|
||||
st := memStore{delivery.PlacementState: &memKV{values: kv.values}}
|
||||
hs := newSeatTools(st, nil)
|
||||
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
|
||||
"placement.anchor": {{Key: "placement.anchor", Value: raw, Revision: 1, Op: "put"}},
|
||||
"anchor.node": {{Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
|
||||
}}
|
||||
pa, err := hs.doPlacement(map[string]any{"node": "anchor", "history": "true"})
|
||||
a, _ := pa.(placementAnswer)
|
||||
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
|
||||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
|
||||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
|
||||
t.Errorf("history = %+v, %v", pa, err)
|
||||
}
|
||||
// No placement: the mirror first, then the node key, then the module keys.
|
||||
kv.log = nil
|
||||
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete placement.anchor delete anchor.node delete anchor.shell" {
|
||||
t.Errorf("the delete = %v, %v", kv.log, err)
|
||||
}
|
||||
})
|
||||
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now())
|
||||
p.kvs[provisions.ProvisionsBucket] = &memKV{}
|
||||
of := p.provisionsOf()
|
||||
if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed {
|
||||
t.Fatalf("provisions = %+v", got)
|
||||
}
|
||||
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now())
|
||||
p.provisionsOf()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings after provided = %v", findings)
|
||||
}
|
||||
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
|
||||
ctx := context.Background()
|
||||
p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
|
||||
gone := map[string]bool{"ace.photos.s3": true}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
|
||||
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if len(findings) != 0 {
|
||||
t.Fatal("said before its bound")
|
||||
}
|
||||
p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
|
||||
t.Fatalf("past its bound: findings %v", findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, map[string]bool{})
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("keyed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
|
||||
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
|
||||
func wantsVerbs(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||
st := memStore{}
|
||||
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
|
||||
// a fingerprint, and laptop's web refused there.
|
||||
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
|
||||
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
|
||||
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
|
||||
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
|
||||
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
|
||||
manifest := func(name, extra string) {
|
||||
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
|
||||
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
|
||||
}
|
||||
manifest("web", "")
|
||||
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
|
||||
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`)
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
var asked []string
|
||||
var said []string
|
||||
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
}, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
|
||||
policyOf := func(module string) policyRow {
|
||||
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(map[string]any)["modules"].([]policyRow)[0]
|
||||
}
|
||||
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
|
||||
t.Errorf("photos = %+v", p)
|
||||
}
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
|
||||
// controller being gone (1126/B8).
|
||||
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
|
||||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
|
||||
t.Errorf("compare = %+v", got)
|
||||
}
|
||||
pub, _ := out.(map[string]any)["published"].(map[string][]publishedRow)
|
||||
if a, l := pub["anchor"], pub["laptop"]; len(a) != 2 || a[0].Module != "node" || a[1].Module != "web" || a[1].FP != fpNew ||
|
||||
!strings.HasPrefix(a[1].Digest, "sha256:") || len(l) != 1 || l[0].Refused != "a required value is unset" || l[0].Digest != "" {
|
||||
t.Errorf("published = %+v (unread %v)", pub, out.(map[string]any)["published-unread"])
|
||||
}
|
||||
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
|
||||
// large states (the modules, their manifests, the composed record).
|
||||
watches := map[string]int{}
|
||||
byWatch := newWantsTools(st, nil, nil)
|
||||
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
watches[state]++
|
||||
for k, v := range st[state].values {
|
||||
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true})
|
||||
}
|
||||
return nil
|
||||
}, func(state, key string) ([]byte, bool, error) {
|
||||
watches[state+" by key"]++
|
||||
v, ok := st[state].values[key]
|
||||
return v, ok, nil
|
||||
})
|
||||
for range 2 {
|
||||
wout, werr := byWatch.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("compare by watch = %+v", got)
|
||||
}
|
||||
}
|
||||
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
|
||||
t.Errorf("the manifests were read by key %d times, want each once", n)
|
||||
}
|
||||
for _, state := range []string{modulesState, haveState, health.FindingsState} {
|
||||
if watches[state] != 1 {
|
||||
t.Errorf("%s was watched %d times, want once", state, watches[state])
|
||||
}
|
||||
}
|
||||
// ADR 0580 clause 3: the compare reads through mesh-sdk's watched reader and the seat keeps no mirror of its own.
|
||||
for f := range reflect.TypeFor[wantsTools]().Fields() {
|
||||
if f.Type.Kind() == reflect.Map || (f.Type.Kind() == reflect.Pointer && f.Type.Elem().Kind() == reflect.Map) {
|
||||
t.Errorf("wantsTools keeps %s, a copy of its own beside the watched reader", f.Name)
|
||||
}
|
||||
}
|
||||
if watches[policiesState] != 0 {
|
||||
t.Errorf("the seat's own small policies state was watched: %v", watches)
|
||||
}
|
||||
// a person's record for web: written, recorded, said.
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chosen, _ := assign.DecodePolicies(st[policiesState].values)
|
||||
if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") ||
|
||||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
|
||||
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
|
||||
t.Error("the bus was given a policy")
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
|
||||
t.Error("a policy without why was set")
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 {
|
||||
t.Errorf("default left %v (%v)", st[policiesState].values, err)
|
||||
}
|
||||
// bus: the node behind the indexer's newest.
|
||||
b, err := w.doBus(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodes := b.(map[string]any)["nodes"].([]busNode)
|
||||
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
|
||||
t.Errorf("bus = %+v, asked %v", b, asked)
|
||||
}
|
||||
if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// readManifest is the module's manifest, read as the controller reads it.
|
||||
func readManifest() (catalogue.Manifest, error) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.Decode(raw)
|
||||
}
|
||||
|
||||
// countingKV is a memKV that counts its puts.
|
||||
type countingKV struct {
|
||||
memKV
|
||||
writes *int
|
||||
}
|
||||
|
||||
func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
*c.writes++
|
||||
return c.memKV.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// revKV is a bucket with real revisions, refusing a write at a revision that is not the key's, and a write of refuse;
|
||||
// log is every write in order.
|
||||
type revKV struct {
|
||||
jetstream.KeyValue
|
||||
rev uint64
|
||||
values map[string][]byte
|
||||
revs map[string]uint64
|
||||
log []string
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (k *revKV) write(key string, value []byte) uint64 {
|
||||
k.rev++
|
||||
k.values[key], k.revs[key] = value, k.rev
|
||||
return k.rev
|
||||
}
|
||||
|
||||
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if _, ok := k.values[key]; ok || key == k.refuse {
|
||||
return 0, errors.New("key exists")
|
||||
}
|
||||
k.log = append(k.log, "create "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if k.revs[key] != revision || key == k.refuse {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "update "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
if key == k.refuse {
|
||||
return errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "delete "+key)
|
||||
delete(k.values, key)
|
||||
delete(k.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// historyStore is a memStore whose history is given.
|
||||
type historyStore struct {
|
||||
memStore
|
||||
revisions map[string][]stdio.StateRevision
|
||||
}
|
||||
|
||||
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
|
||||
return s.revisions[key], nil
|
||||
}
|
||||
@@ -18,8 +18,8 @@ import (
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
@@ -351,7 +351,14 @@ func serveTools() error {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
return stdio.Serve("", s.tools())
|
||||
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
|
||||
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
|
||||
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
|
||||
return stdio.SeatPublish(wantsEventSubject(e), b, id)
|
||||
})
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
tools = append(tools, wants.tools()...)
|
||||
return stdio.Serve("", tools)
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
busStepCause = "bus-upgrade"
|
||||
)
|
||||
|
||||
// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit.
|
||||
type wantsTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
emit func(event, id string, body any) error
|
||||
now func() time.Time
|
||||
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
|
||||
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
|
||||
// a large state (issue 1953 L5f). Nil reads key by key.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
// wantsEventSubject is the subject of one of the seat's events.
|
||||
func wantsEventSubject(event string) string {
|
||||
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
|
||||
}
|
||||
|
||||
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Values(state)
|
||||
}
|
||||
return s.direct(state)
|
||||
}
|
||||
|
||||
// direct reads a state key by key, for a small state of the seat's own.
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
|
||||
// under its digest.
|
||||
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Once(manifestsState, digest)
|
||||
}
|
||||
raw, _, found, err := s.st.get(manifestsState, digest)
|
||||
return raw, found, err
|
||||
}
|
||||
|
||||
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
|
||||
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
|
||||
mods, err := s.values(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
|
||||
for name, raw := range mods {
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
held[name] = m
|
||||
if m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mraw, found, err := s.manifest(m.Manifest)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(mraw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
|
||||
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
|
||||
out[name] = man
|
||||
}
|
||||
}
|
||||
return out, held, nil
|
||||
}
|
||||
|
||||
// policyRow is one module's policy as upgrade answers it.
|
||||
type policyRow struct {
|
||||
Module string `json:"module"`
|
||||
Policy string `json:"policy"`
|
||||
From string `json:"from"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
|
||||
// back with default; with compare each node's composition and the assigner's findings.
|
||||
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
manifests, _, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
}
|
||||
out := map[string]any{"composed": composed}
|
||||
// And what every node published (ADR 0593 §8), read only to compare.
|
||||
if values, err := s.values(delivery.ComposedRead); err != nil {
|
||||
out["published-unread"] = err.Error()
|
||||
} else {
|
||||
out["published"] = published(values)
|
||||
}
|
||||
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
} else {
|
||||
keys := []string{}
|
||||
for k := range found {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out["findings"] = keys
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
module, policy := arg(args, "module"), arg(args, "policy")
|
||||
if policy != "" {
|
||||
return s.setPolicy(module, policy, arg(args, "why"), by, manifests)
|
||||
}
|
||||
var rows []policyRow
|
||||
for name, m := range manifests {
|
||||
if module != "" && name != module {
|
||||
continue
|
||||
}
|
||||
p, from, why, who := assign.PolicyOf(m, chosen)
|
||||
rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who})
|
||||
}
|
||||
if module != "" && len(rows) == 0 {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
|
||||
return map[string]any{"modules": rows, "unreadable": unreadable}, nil
|
||||
}
|
||||
|
||||
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
|
||||
// and says policy-changed. The bus is never rolled: a choice for it is refused.
|
||||
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("a policy is set for one module: name it")
|
||||
}
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
if catalogue.ProvidesBus(m) {
|
||||
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("a person's policy says why")
|
||||
}
|
||||
if by == "" || by == stdio.Unproven {
|
||||
return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller")
|
||||
}
|
||||
what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy)
|
||||
if policy == "default" {
|
||||
if err := s.st.del(policiesState, module); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err)
|
||||
}
|
||||
} else {
|
||||
p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()}
|
||||
if err := p.Check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _ := json.Marshal(p)
|
||||
if err := s.st.put(policiesState, module, raw); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := handacts.Recorder{Ask: s.ask}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
recorded = "not recorded: " + rerr.Error()
|
||||
}
|
||||
body := map[string]any{"module": module, "policy": policy, "why": why, "by": by}
|
||||
said := ""
|
||||
if s.emit != nil {
|
||||
id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10)
|
||||
if err := s.emit(PolicyChangedEvent, id, body); err != nil {
|
||||
said = "not said: " + err.Error()
|
||||
}
|
||||
}
|
||||
return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil
|
||||
}
|
||||
|
||||
// busNode is one node's bus as bus answers it.
|
||||
type busNode struct {
|
||||
Node string `json:"node"`
|
||||
FP string `json:"fp"`
|
||||
Behind bool `json:"behind"`
|
||||
}
|
||||
|
||||
// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind,
|
||||
// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only.
|
||||
func (s *wantsTools) doBus(map[string]any) (any, error) {
|
||||
manifests, held, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus := ""
|
||||
for name, m := range manifests {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
bus = name
|
||||
}
|
||||
}
|
||||
if bus == "" {
|
||||
return nil, errors.New("the indexer holds no module that provides the bus")
|
||||
}
|
||||
newest := held[bus].FP2
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nodes []busNode
|
||||
for key, raw := range haves {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode || module != bus {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHave(raw)
|
||||
if err != nil || h.Phase == delivery.PhaseRemoved {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
|
||||
}
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
|
||||
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
|
||||
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
|
||||
out["steps-unread"] = err.Error()
|
||||
} else {
|
||||
out["steps"] = steps
|
||||
}
|
||||
puts, err := s.values(putsState)
|
||||
if err != nil {
|
||||
out["puts-unread"] = err.Error()
|
||||
} else {
|
||||
mine := map[string]json.RawMessage{}
|
||||
for k, v := range puts {
|
||||
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
|
||||
mine[k] = v
|
||||
}
|
||||
}
|
||||
out["puts"] = mine
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
|
||||
func (s *wantsTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
|
||||
"bus": s.doBus,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,12 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -15,7 +12,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -24,18 +20,6 @@ import (
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -46,6 +30,19 @@ type world struct {
|
||||
healthy map[string][]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
|
||||
trunkBuilt map[string]delivery.BuildLogEntry
|
||||
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
|
||||
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
placement *delivery.PlacementKeys
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
type entryMeta struct {
|
||||
revision uint64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -56,7 +53,30 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
|
||||
w.put(bucket, key, value)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(w.meta, bucket+"|"+key)
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
}
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
func (w *world) metaOf(bucket, key string) entryMeta {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.meta[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -66,12 +86,20 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
if w.values[bucket] == nil {
|
||||
w.values[bucket] = map[string][]byte{}
|
||||
}
|
||||
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
|
||||
if value == nil {
|
||||
delete(w.values[bucket], key)
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, nil, 0, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.PlacementBucket {
|
||||
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
|
||||
_, _ = w.placement.Put(key, value, 0, false)
|
||||
}
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -149,10 +177,10 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
m, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
p = parsed{err: err.Error()}
|
||||
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
|
||||
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
|
||||
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
|
||||
p = parsed{module: m.Module, err: err.Error()}
|
||||
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
|
||||
// module's value where it is read (world.mesh).
|
||||
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
|
||||
} else {
|
||||
p = parsed{module: m.Module, manifest: man}
|
||||
}
|
||||
@@ -162,21 +190,42 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
return out
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
for node, p := range w.placements() {
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, all
|
||||
return out, len(w.read(delivery.PlacementBucket)) > 0
|
||||
}
|
||||
|
||||
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
|
||||
func (w *world) placements() map[string]delivery.Placement {
|
||||
out, _ := w.placement.AllRevisions()
|
||||
return out
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
@@ -194,75 +243,35 @@ func (w *world) trunk() map[string]delivery.Module {
|
||||
return out
|
||||
}
|
||||
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned, _ = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
|
||||
// with those placement names (ADR 0593 §8).
|
||||
func (w *world) records() map[string]delivery.NodeRecord {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
m.Digests[name] = mod.Manifest
|
||||
p := w.manifest(ctx, manifests, mod.Manifest)
|
||||
if p.err != "" {
|
||||
m.Unreadable[name] = p.err
|
||||
continue
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
return m
|
||||
return out
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
|
||||
definedIn := map[string]map[string]bool{}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository == assign.SDKRepository {
|
||||
ch.SDKHead = p.Head.Head
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
ch.Members[p.Repository] = p.Head.Head
|
||||
for _, name := range p.Modules {
|
||||
fp, ok := p.Fps[name]
|
||||
if !ok || fp.Manifest == "" {
|
||||
if ok && fp.None2 == noModuleWords {
|
||||
ch.Removed = append(ch.Removed, name)
|
||||
continue
|
||||
}
|
||||
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
|
||||
}
|
||||
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
|
||||
ch.Rebuilt++
|
||||
}
|
||||
got := w.manifest(ctx, manifests, fp.Manifest)
|
||||
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
|
||||
return ch, got.err
|
||||
}
|
||||
ch.Digests[name] = fp.Manifest
|
||||
if got.err != "" {
|
||||
ch.Unreadable[name] = got.err
|
||||
continue
|
||||
}
|
||||
ch.Manifests[got.module] = got.manifest
|
||||
if definedIn[got.module] == nil {
|
||||
definedIn[got.module] = map[string]bool{}
|
||||
}
|
||||
definedIn[got.module][p.Repository] = true
|
||||
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
|
||||
definedIn[got.module][t.Repository] = true
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
}
|
||||
}
|
||||
for name, repos := range definedIn {
|
||||
if len(repos) > 1 {
|
||||
for r := range repos {
|
||||
ch.Twice[name] = append(ch.Twice[name], r)
|
||||
}
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
return ch, ""
|
||||
return out
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
@@ -333,6 +342,10 @@ func (w *world) value(bucket, key string) []byte {
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
|
||||
// manifest but the controller's reads the controller's grants.
|
||||
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
@@ -357,36 +370,14 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
@@ -401,6 +392,14 @@ func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// bucketGen is how many times any key of the bucket was written or deleted in this process: a read of what it shows
|
||||
// elsewhere is asked again only after it moves.
|
||||
func (w *world) bucketGen(bucket string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket]
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
@@ -408,29 +407,35 @@ func (w *world) changed(bucket, key string) uint64 {
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
|
||||
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
|
||||
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
|
||||
_, all := w.assigned()
|
||||
seen := map[string]bool{}
|
||||
for _, a := range all {
|
||||
if a.Module != module || a.Build == "" {
|
||||
continue
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, a.Build))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
|
||||
continue
|
||||
}
|
||||
seen[rec.Commit] = true
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for c := range seen {
|
||||
out = append(out, c)
|
||||
raw, ok := w.read(delivery.HaveBucket)[key]
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil || h.Composed == "" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err == nil && isNode && !placed[node] {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
|
||||
@@ -2,14 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// fakeManifests is the indexer's manifests bucket in memory.
|
||||
@@ -44,76 +42,8 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
|
||||
return p
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
|
||||
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
|
||||
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
|
||||
pull(assign.SDKRepository, 3, nil),
|
||||
}}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
|
||||
if waits != "" {
|
||||
t.Fatalf("the change waits: %s", waits)
|
||||
}
|
||||
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
|
||||
t.Errorf("change = %+v", ch)
|
||||
}
|
||||
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
|
||||
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
|
||||
// saying for what; an unreadable manifest is the change's, not a wait.
|
||||
func TestAChangeWaitsForItsManifests(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
|
||||
for _, c := range []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
|
||||
} {
|
||||
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
|
||||
if waits == "" {
|
||||
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
|
||||
}
|
||||
}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["web"] == "" {
|
||||
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
|
||||
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
|
||||
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
|
||||
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
|
||||
// xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"predefined"`) {
|
||||
t.Fatal("the fixture names no predefined key")
|
||||
}
|
||||
f := fakeManifests{}
|
||||
xterm := f.add(t, "xterm", string(raw))
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
@@ -124,4 +54,27 @@ func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
inPart bool
|
||||
granted bool
|
||||
}{
|
||||
{"the part's grant", true, true},
|
||||
{"no grant", false, false},
|
||||
} {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
|
||||
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
|
||||
}
|
||||
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
|
||||
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -81,6 +81,29 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
@@ -96,6 +119,14 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
@@ -103,6 +134,17 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||
@@ -73,7 +75,13 @@ func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
@@ -87,7 +95,7 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
@@ -97,6 +105,49 @@ func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
// findings bucket cannot be written to at all.
|
||||
func saidConditionsAreFindings(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
raised, cleared asks.Condition
|
||||
key string
|
||||
}{
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
|
||||
f := Findings{Put: func(k string, v []byte) error {
|
||||
got, err := health.DecodeFinding(v)
|
||||
puts[k] = got
|
||||
return err
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(asks.Condition) error { said++; return nil })
|
||||
if err := say(c.raised); err != nil {
|
||||
t.Fatalf("%s: %v", c.key, err)
|
||||
}
|
||||
got, ok := puts[c.key]
|
||||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
|
||||
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
|
||||
}
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
|
||||
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
if err := (Findings{}).Say(func(asks.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
|
||||
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
|
||||
t.Error("a condition not the assigner's is a finding")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
|
||||
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
|
||||
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
|
||||
// and writes nothing itself: the program hands it what it watched and writes what it answers.
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
|
||||
const SDKRepository = "novox/mesh-sdk"
|
||||
|
||||
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
|
||||
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
|
||||
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
|
||||
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
|
||||
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
|
||||
"node-engine's validator would refuse"
|
||||
|
||||
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
|
||||
// why, and each node's open assignments.
|
||||
type Mesh struct {
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Assigned map[string][]string
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
|
||||
// member (empty when none), and how many modules it rebuilds.
|
||||
type Change struct {
|
||||
Group, Heads string
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Removed []string
|
||||
Twice map[string][]string
|
||||
SDKHead string
|
||||
Rebuilt int
|
||||
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine.
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
moved[name] = true
|
||||
}
|
||||
for name := range c.Unreadable {
|
||||
moved[name] = true
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
moved[name] = true
|
||||
}
|
||||
provided, required := map[string]bool{}, map[string]bool{}
|
||||
for name := range moved {
|
||||
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
|
||||
for _, o := range man.Provides {
|
||||
provided[o.Name] = true
|
||||
}
|
||||
}
|
||||
for _, r := range c.Manifests[name].Requires {
|
||||
required[r] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
|
||||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
|
||||
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
|
||||
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
|
||||
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
|
||||
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
|
||||
in := compose.Input{Catalogue: catalog}
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
for _, mod := range assigned[n] {
|
||||
for _, cl := range catalog[mod].Claims {
|
||||
scope := cl.Scope
|
||||
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||
scope = seat.Scope
|
||||
}
|
||||
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
|
||||
}
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
|
||||
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
|
||||
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
|
||||
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
|
||||
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
|
||||
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
|
||||
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
|
||||
var fails []string
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Unreadable) {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
trunk := input(m.Manifests, m.Assigned)
|
||||
changed := map[string]catalogue.Manifest{}
|
||||
for name, man := range m.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for name, man := range c.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
switch {
|
||||
case now == nil:
|
||||
case before != nil:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
|
||||
}
|
||||
}
|
||||
if c.SDKHead != "" {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
|
||||
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
|
||||
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
|
||||
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
|
||||
if c.Rebuilt > delivery.CompositionRebuildWarning {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
|
||||
}
|
||||
v.Warnings = append(v.Warnings, NotJudged)
|
||||
if could := CouldRun(m, c, machines); len(could) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
|
||||
}
|
||||
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
|
||||
}
|
||||
if len(fails) > 0 {
|
||||
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
|
||||
return v
|
||||
}
|
||||
v.State = delivery.CompositionPass
|
||||
return v
|
||||
}
|
||||
|
||||
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
|
||||
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
|
||||
// the facts snapshot), so each is named as not judged.
|
||||
func CouldRun(m Mesh, c Change, composed []string) []string {
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for name := range c.Manifests {
|
||||
if !slices.Contains(modules, name) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
|
||||
func grantsOf(man catalogue.Manifest) string {
|
||||
var offers []string
|
||||
for _, o := range man.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
|
||||
}
|
||||
|
||||
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
|
||||
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
|
||||
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
|
||||
func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
provisions := map[string]bool{}
|
||||
altered := false
|
||||
for name, man := range c.Manifests {
|
||||
before, had := m.Manifests[name]
|
||||
if had && grantsOf(before) == grantsOf(man) {
|
||||
continue
|
||||
}
|
||||
altered = true
|
||||
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
|
||||
provisions[r] = true
|
||||
}
|
||||
}
|
||||
if !altered {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func meshOf() Mesh {
|
||||
return Mesh{
|
||||
Manifests: map[string]catalogue.Manifest{
|
||||
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
|
||||
"web": {Module: "web", Version: "1"},
|
||||
},
|
||||
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
|
||||
}
|
||||
}
|
||||
|
||||
func hasWarning(v delivery.CompositionVerdict, part string) bool {
|
||||
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
|
||||
}
|
||||
|
||||
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
|
||||
// the machine and the mesh-sdk it composed with, and says what it did not judge.
|
||||
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "v0.1.14-x", 0)
|
||||
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
|
||||
}
|
||||
if !hasWarning(v, "issue 1461") {
|
||||
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||
// consumer's machine is touched when its provider changes.
|
||||
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
|
||||
func TestUnreadableTwiceAndWide(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
|
||||
v = Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
|
||||
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
|
||||
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
|
||||
v := Compose(meshOf(), c, "v0.1.14-own", 0)
|
||||
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
|
||||
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
|
||||
}
|
||||
c.SDKHead = ""
|
||||
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
|
||||
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
|
||||
// assigned nowhere touches none, and the verdict says so.
|
||||
func TestTouched(t *testing.T) {
|
||||
m := meshOf()
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
|
||||
t.Errorf("touched by store = %v", got)
|
||||
}
|
||||
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
|
||||
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
|
||||
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
|
||||
// names none of the second.
|
||||
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
|
||||
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
|
||||
m.Assigned["novox"] = []string{"bus"}
|
||||
m.Assigned["shanks"] = []string{"db"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
|
||||
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
|
||||
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
|
||||
}
|
||||
if !hasWarning(v, "whose grants alone the change alters: novox") {
|
||||
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
|
||||
}
|
||||
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
|
||||
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
|
||||
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
|
||||
const (
|
||||
CaseTrunk = "trunk"
|
||||
CaseChange = "change"
|
||||
)
|
||||
|
||||
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
|
||||
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
|
||||
const SDKDir = "go"
|
||||
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
|
||||
// them, the node-engine applies them. Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
|
||||
|
||||
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
|
||||
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
|
||||
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
|
||||
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
|
||||
for name, digest := range m.Digests {
|
||||
trunk[name] = composer.Manifest{Digest: digest}
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for name, digest := range c.Digests {
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
|
||||
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
|
||||
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
|
||||
changed := map[string]composer.Manifest{}
|
||||
for name, digest := range c.Digests {
|
||||
changed[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
|
||||
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
|
||||
}
|
||||
|
||||
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return delivery.ComposeJob{}, err
|
||||
}
|
||||
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
|
||||
Package: composer.Package, Request: raw, Reader: reader}, nil
|
||||
}
|
||||
|
||||
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
|
||||
// composer derives the seats held from the claims, as input does).
|
||||
func plainInput(assigned map[string][]string) compose.Input {
|
||||
var in compose.Input
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
|
||||
type Ran struct {
|
||||
State string
|
||||
Reason string
|
||||
Verdict *composer.Verdict
|
||||
}
|
||||
|
||||
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
|
||||
// failure, named.
|
||||
func RanOf(step delivery.Step) Ran {
|
||||
r := Ran{State: step.State, Reason: step.Reason}
|
||||
if step.State != delivery.StepPassed {
|
||||
return r
|
||||
}
|
||||
var a delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &a); err != nil {
|
||||
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
|
||||
}
|
||||
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
|
||||
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
|
||||
}
|
||||
r.Verdict = &v
|
||||
return r
|
||||
}
|
||||
|
||||
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
|
||||
if v == nil {
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name == name {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
|
||||
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
|
||||
// the group holds one in its repository, else a build its node runs.
|
||||
type Reader struct {
|
||||
Module, Commit string
|
||||
Member bool
|
||||
Ran Ran
|
||||
}
|
||||
|
||||
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
|
||||
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
|
||||
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
|
||||
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
|
||||
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
|
||||
// said and judged nothing.
|
||||
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
var fails []string
|
||||
if head.State != delivery.StepPassed {
|
||||
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
|
||||
v.State = delivery.CompositionFail
|
||||
return v
|
||||
}
|
||||
if head.Verdict.SDK != "" {
|
||||
v.SDK = head.Verdict.SDK
|
||||
}
|
||||
trunk, okT := caseOf(head.Verdict, CaseTrunk)
|
||||
change, okC := caseOf(head.Verdict, CaseChange)
|
||||
if !okT || !okC {
|
||||
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
|
||||
return v
|
||||
}
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if r := change.Manifests[name]; !r.Accepted {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(trunk.Manifests) {
|
||||
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
if _, before := m.Unreadable[name]; before {
|
||||
continue // already unreadable on the trunk: not the group's doing
|
||||
}
|
||||
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
before, now := trunk.Machines[node], change.Machines[node]
|
||||
switch {
|
||||
case now.Composes:
|
||||
case !before.Composes:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
|
||||
}
|
||||
}
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
|
||||
// from before issue 1621): Go's words for a package its module does not hold.
|
||||
func NoComposer(reason string) bool {
|
||||
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
|
||||
if strings.Contains(reason, words) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
|
||||
for _, r := range readers {
|
||||
at := r.Module + " at " + r.Commit
|
||||
if r.Member {
|
||||
at = r.Module + "'s member head " + r.Commit
|
||||
}
|
||||
if r.Ran.State != delivery.StepPassed {
|
||||
if NoComposer(r.Ran.Reason) {
|
||||
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
|
||||
"composer command (before issue 1621)", at))
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
|
||||
continue
|
||||
}
|
||||
read, ok := caseOf(r.Ran.Verdict, CaseChange)
|
||||
if !ok {
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
|
||||
continue
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
|
||||
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
|
||||
}
|
||||
}
|
||||
return fails, warnings
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
const sdkHead = "1111111111111111111111111111111111111111"
|
||||
|
||||
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
|
||||
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
|
||||
var raws = map[string]string{
|
||||
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
|
||||
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
|
||||
"d-web": `{"module": "web", "version": "1"}`,
|
||||
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
|
||||
}
|
||||
|
||||
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
|
||||
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
|
||||
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
|
||||
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
|
||||
|
||||
func digestedMesh() Mesh {
|
||||
m := meshOf()
|
||||
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
|
||||
return m
|
||||
}
|
||||
|
||||
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
|
||||
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
|
||||
func webShares(members map[string]string) Change {
|
||||
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
|
||||
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
|
||||
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
|
||||
}
|
||||
|
||||
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
|
||||
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
|
||||
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
|
||||
t.Helper()
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := composer.Fill(&req, func(d string) ([]byte, error) {
|
||||
if r, ok := read[d]; ok {
|
||||
return []byte(r), nil
|
||||
}
|
||||
return []byte(raws[d]), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.SDK = "(devel) " + job.Fetch["commit"].(string)
|
||||
vr, _ := json.Marshal(v)
|
||||
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
|
||||
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
|
||||
}
|
||||
|
||||
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
|
||||
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
|
||||
// change by digest, never the manifests' bytes, composing the machines the change touches.
|
||||
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, err := HeadJob(digestedMesh(), c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
|
||||
t.Fatalf("job = %+v", job)
|
||||
}
|
||||
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: testTime, Compose: &job}
|
||||
if _, err := delivery.EncodeStepAsk(ask); err != nil {
|
||||
t.Fatalf("the ask is refused by its layout: %v", err)
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
|
||||
t.Fatalf("cases = %+v", req.Cases)
|
||||
}
|
||||
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
|
||||
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
|
||||
}
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
|
||||
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
|
||||
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
v := FromComposer(digestedMesh(), c, head, nil, 0)
|
||||
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
|
||||
t.Fatalf("with the head's composer: %+v", v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
|
||||
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
|
||||
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
|
||||
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
|
||||
// the SDK head (its composer reads what the head reads) it passes.
|
||||
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
running := "3333333333333333333333333333333333333333"
|
||||
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
|
||||
t.Fatalf("reader job = %+v", rjob)
|
||||
}
|
||||
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
|
||||
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
|
||||
}
|
||||
member := "2222222222222222222222222222222222222222"
|
||||
pinned := run(t, rjob, asTheHeadReads)
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
|
||||
if v.State != delivery.CompositionPass {
|
||||
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
|
||||
// failed for another cause fails the group, loud.
|
||||
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
|
||||
t.Fatalf("a reader with no composer: %+v", v)
|
||||
}
|
||||
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
|
||||
t.Fatalf("a reader whose composer failed: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
|
||||
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
|
||||
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
|
||||
t.Fatalf("a head composer that did not run: %+v", v)
|
||||
}
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
|
||||
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
|
||||
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
|
||||
// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair
|
||||
// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its
|
||||
// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router
|
||||
// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the
|
||||
// assigner no longer says it straight (never both).
|
||||
|
||||
// FindingsBucket is the assigner's findings.
|
||||
var FindingsBucket = health.FindingsBucket(delivery.AssignerModule)
|
||||
|
||||
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
|
||||
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
|
||||
// a clear, whose key is to be deleted; value is nil then.
|
||||
func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
|
||||
rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".")
|
||||
if !ok || rest == "" {
|
||||
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
|
||||
}
|
||||
kind, id, _ := strings.Cut(rest, ".")
|
||||
if id == "" {
|
||||
id = c.Subject.ID
|
||||
}
|
||||
scope := c.Subject.Scope
|
||||
if scope == "" {
|
||||
scope = health.ScopeModule
|
||||
}
|
||||
key = health.FindingKey(scope, id, kind)
|
||||
if c.State == asks.StateCleared {
|
||||
return key, nil, false, nil
|
||||
}
|
||||
if c.State != asks.StateRaised {
|
||||
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
|
||||
}
|
||||
summary := c.Summary
|
||||
if summary == "" {
|
||||
summary = c.Headline
|
||||
}
|
||||
var actions []health.Action
|
||||
for _, a := range c.Actions {
|
||||
actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
|
||||
}
|
||||
value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine},
|
||||
Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved,
|
||||
Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind,
|
||||
Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
return key, value, true, err
|
||||
}
|
||||
|
||||
// Findings is where the assigner's findings go: the bucket's put and delete.
|
||||
type Findings struct {
|
||||
Put func(key string, value []byte) error
|
||||
Delete func(key string) error
|
||||
Keys func() ([]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
|
||||
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
|
||||
// findings bucket at all. A finding that cannot be written is the error.
|
||||
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
var errs []error
|
||||
if f.Put != nil {
|
||||
now := time.Now
|
||||
if f.Now != nil {
|
||||
now = f.Now
|
||||
}
|
||||
key, value, raised, err := FindingOf(c, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
errs = append(errs, err)
|
||||
case raised:
|
||||
if err := f.Put(key, value); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
|
||||
}
|
||||
default:
|
||||
if err := f.Delete(key); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.Put == nil {
|
||||
if say != nil {
|
||||
if err := say(c); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
key := delivery.AssignerModule + ".seat-states"
|
||||
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
|
||||
if len(unbound) == 0 {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
|
||||
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
|
||||
|
||||
// PoliciesBucket is the assigner's state `policies`.
|
||||
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
|
||||
|
||||
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
|
||||
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
|
||||
out := map[string]catalogue.PersonPolicy{}
|
||||
var bad []string
|
||||
for key, raw := range values {
|
||||
var p catalogue.PersonPolicy
|
||||
if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key {
|
||||
bad = append(bad, key)
|
||||
continue
|
||||
}
|
||||
out[key] = p
|
||||
}
|
||||
sort.Strings(bad)
|
||||
return out, bad
|
||||
}
|
||||
|
||||
// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one.
|
||||
func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) {
|
||||
var c *catalogue.PersonPolicy
|
||||
if p, ok := chosen[m.Module]; ok {
|
||||
c = &p
|
||||
}
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
|
||||
const KindProvisionFailed = "provision-failed"
|
||||
|
||||
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
|
||||
func ProvisionFailedKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed)
|
||||
}
|
||||
|
||||
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
|
||||
func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
|
||||
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
|
||||
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.",
|
||||
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}},
|
||||
Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func also(provider, consumer string) []string {
|
||||
if provider == "" || provider == consumer {
|
||||
return nil
|
||||
}
|
||||
return []string{provider}
|
||||
}
|
||||
|
||||
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
|
||||
const KindProvisionMissing = "provision-missing"
|
||||
|
||||
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
|
||||
func ProvisionMissingKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
|
||||
}
|
||||
|
||||
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
|
||||
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
|
||||
parts := strings.SplitN(key, ".", 3)
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
|
||||
}
|
||||
node, module, provision := parts[0], parts[1], parts[2]
|
||||
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
|
||||
"no provider of it may be placed", module, node, provision, bound, module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
|
||||
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.",
|
||||
Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6).
|
||||
const KindUnplaced = "unplaced"
|
||||
|
||||
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
|
||||
func UnplacedKey(node string) string {
|
||||
return health.FindingKey(health.ScopeMachine, node, KindUnplaced)
|
||||
}
|
||||
|
||||
// UnplacedFinding is the finding of a node no placement names since the lease write.
|
||||
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+
|
||||
"so the assigner writes it no want", node)
|
||||
return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced,
|
||||
Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary,
|
||||
Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.",
|
||||
Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".",
|
||||
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func clipHeadline(s string) string {
|
||||
if len([]rune(s)) <= 60 {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:60])
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func StepPut(w PutWorld) PutNext {
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
|
||||
Warrant *delivery.AddNodeWarrant
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
}
|
||||
if p.Why == "" {
|
||||
findings = append(findings, "a proposal says why: it is shown to the operator")
|
||||
}
|
||||
if len(in.Changes) == 0 {
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
}
|
||||
out := map[string]*delivery.Placement{}
|
||||
for node, pl := range laid {
|
||||
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
|
||||
continue
|
||||
}
|
||||
pl := pl
|
||||
all[node] = pl
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
|
||||
// candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
|
||||
for _, node := range p.RemovedNodes() {
|
||||
if _, ok := in.Current[node]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(all, node)
|
||||
out[node] = nil
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
held, recorded := in.Held, in.Recorded
|
||||
if held == nil {
|
||||
held = func(string) bool { return false }
|
||||
}
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
// A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by
|
||||
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
|
||||
joining := map[string]bool{}
|
||||
for _, c := range in.Changes {
|
||||
if c.Op == delivery.OpAddNode {
|
||||
joining[c.Node] = true
|
||||
}
|
||||
}
|
||||
if len(joining) > 0 {
|
||||
was := recorded
|
||||
recorded = func(node string) bool { return joining[node] || was(node) }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// ControllerModule is the controller's module, which nothing places again (issue 1126).
|
||||
const ControllerModule = "mesh-controller"
|
||||
|
||||
// ControllerEnded is why a placement of the controller is refused: the parts that acted only once its lease was handed
|
||||
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
|
||||
// would write beside them, silently.
|
||||
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
|
||||
|
||||
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
|
||||
// whole placement, or in what the changes lay. None when nothing does.
|
||||
func ControllerRefusals(changes []delivery.PlacementProposalChange, laid map[string]delivery.Placement) []string {
|
||||
var out []string
|
||||
nodes := map[string]bool{}
|
||||
for i, c := range changes {
|
||||
places := c.Module == ControllerModule && c.Op != delivery.OpTakeOff
|
||||
if c.Whole != nil && slices.Contains(c.Whole.Assigned(), ControllerModule) {
|
||||
places = true
|
||||
}
|
||||
if places {
|
||||
out = append(out, fmt.Sprintf("change %d places %s on %s: %s", i+1, ControllerModule, c.Node, ControllerEnded))
|
||||
nodes[c.Node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(laid)) {
|
||||
if !nodes[node] && slices.Contains(laid[node].Assigned(), ControllerModule) {
|
||||
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
|
||||
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
|
||||
// it is created) and its value (nil: it is deleted).
|
||||
type Write struct {
|
||||
Node, Key string
|
||||
Base uint64
|
||||
Value *delivery.Placement
|
||||
}
|
||||
|
||||
// Writes is every key a proposal writes, by node order.
|
||||
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
|
||||
var out []Write
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
|
||||
// the controller's verbs (ADR 0567 §8, issue 774).
|
||||
const (
|
||||
KindControllerCall = "controller-call"
|
||||
KindControllerPin = "controller-pin"
|
||||
)
|
||||
|
||||
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
|
||||
// module, or node and provision for a pin).
|
||||
func CallClearedCondition(kind, node, module string) asks.Condition {
|
||||
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
|
||||
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
|
||||
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
func PlacementAct(p delivery.PlacementProposal) asks.Act {
|
||||
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
|
||||
}
|
||||
|
||||
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
|
||||
// another asking, under an id of its own.
|
||||
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
|
||||
return int(at.Unix())
|
||||
}
|
||||
|
||||
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
|
||||
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
|
||||
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
|
||||
binds, err := asks.ActDigest(PlacementAct(p))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: headline,
|
||||
Explanation: brief,
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
|
||||
Level: asks.Approve, Binds: binds},
|
||||
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts),
|
||||
OnExpiry: "nothing is written, and the proposal ends",
|
||||
About: key,
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
rest := p
|
||||
rest.Changes, rest.Base = nil, map[string]uint64{}
|
||||
for _, c := range p.Changes {
|
||||
if _, done := written[c.Node]; !done {
|
||||
rest.Changes = append(rest.Changes, c)
|
||||
}
|
||||
}
|
||||
for node, base := range p.Base {
|
||||
if _, done := written[node]; !done {
|
||||
rest.Base[node] = base
|
||||
}
|
||||
}
|
||||
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
|
||||
return rest, len(rest.Changes) > 0
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
|
||||
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
|
||||
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
|
||||
func TestThePlacementSeatDecides(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
|
||||
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
|
||||
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
|
||||
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
|
||||
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
|
||||
Held: func(m string) bool { return m != "ghost" },
|
||||
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
|
||||
}
|
||||
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
|
||||
p, laid, findings := Propose(in(move...))
|
||||
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
|
||||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
|
||||
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
|
||||
}
|
||||
if _, err := delivery.EncodePlacementProposal(p); err != nil {
|
||||
t.Fatalf("the proposal cannot be written: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
|
||||
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
|
||||
// act without its lease.
|
||||
"the controller placed again": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
|
||||
"the controller seeded whole": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "anchor", Whole: &delivery.Placement{V: delivery.Version, Node: "anchor",
|
||||
Decided: anchor.Decided, Modules: []delivery.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("%s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
|
||||
t.Errorf("the rest = %+v, %v", rest, ok)
|
||||
}
|
||||
if _, ok := Rest(p, laid); ok {
|
||||
t.Error("a proposal written whole leaves something to write")
|
||||
}
|
||||
|
||||
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
|
||||
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
|
||||
c := SeatWaitsCondition(unbound)
|
||||
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
|
||||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
|
||||
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
|
||||
}
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
q := p
|
||||
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||
@@ -28,9 +30,17 @@ type Module struct {
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
|
||||
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
|
||||
Provisions func(node string) map[string]provisions.Provision
|
||||
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
||||
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
||||
Answering func(provision string) []catalogue.Manifest
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
@@ -46,8 +56,14 @@ type Module struct {
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
||||
Self func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
@@ -82,14 +98,15 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
|
||||
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3).
|
||||
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
|
||||
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
|
||||
// down and one module never blocks another.
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
|
||||
// without which there is nothing to want: the build done, not held, the grant and the composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
@@ -105,11 +122,6 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
@@ -119,11 +131,58 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
||||
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
||||
func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
if m.Provisions == nil {
|
||||
return "", false
|
||||
}
|
||||
if missing := m.MissingProvisions(node); len(missing) > 0 {
|
||||
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
||||
}
|
||||
keyed := m.Provisions(node)
|
||||
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
||||
if p := keyed[key]; p.State != provisions.Provided {
|
||||
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
||||
if p.Why != "" {
|
||||
words += ": " + p.Why
|
||||
}
|
||||
return words, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
|
||||
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
|
||||
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
|
||||
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
|
||||
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
|
||||
func (m Module) MissingProvisions(node string) []string {
|
||||
if m.Provisions == nil || m.Answering == nil {
|
||||
return nil
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, p := range m.Provisions(node) {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
var out []string
|
||||
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
|
||||
if !have[r] {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
||||
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
||||
// offers it) and holds.
|
||||
@@ -150,6 +209,17 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
||||
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
||||
}
|
||||
|
||||
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
|
||||
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
|
||||
func unreached(m Module, node string, n Nodes) string {
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return why
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
||||
// valuable or rebuildable; cache and none do not).
|
||||
func keepsData(m catalogue.Manifest) bool {
|
||||
@@ -169,6 +239,9 @@ type Outcome struct {
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
@@ -176,6 +249,9 @@ type Outcome struct {
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
|
||||
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
|
||||
Unreached map[string]string
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
@@ -192,10 +268,27 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||
// running an old-form build, got no want and the pass said nothing).
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
Bus bool
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||
func (h Holding) Reason() string {
|
||||
if h.Bus {
|
||||
return h.Words
|
||||
}
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||
func BusBarWords(node, prev, fp string) string {
|
||||
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
@@ -211,11 +304,25 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.FP != m.FP || !failed(h) {
|
||||
continue
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
@@ -260,26 +367,40 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
|
||||
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
|
||||
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
|
||||
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
|
||||
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
|
||||
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
|
||||
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
|
||||
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
if older {
|
||||
_, composed := m.composition(node, have.FP)
|
||||
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
|
||||
}
|
||||
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
switch {
|
||||
case older:
|
||||
ok, target = true, have.FP
|
||||
case oldForm:
|
||||
// held back by rule 3 as written: on neither list until the canary runs F healthy
|
||||
default:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
}
|
||||
}
|
||||
if why := unreached(m, node, n); why != "" {
|
||||
if out.Unreached == nil {
|
||||
out.Unreached = map[string]string{}
|
||||
}
|
||||
out.Unreached[node] = why
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
||||
n.Haves[node][m.Name].FP != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
@@ -291,7 +412,8 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -303,6 +425,14 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||
out.Rules[node] = h
|
||||
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = h
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -368,6 +498,10 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
}
|
||||
}
|
||||
next, write := m.selfDerived(node, before, had, next, n, &out)
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
@@ -380,6 +514,43 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
|
||||
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
||||
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
||||
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
||||
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
||||
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
||||
// while its ask waits, the want written stays and the node's holding says why.
|
||||
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
|
||||
switch {
|
||||
case next.Removed:
|
||||
return next, true
|
||||
case m.selfOn(node):
|
||||
next.Self, next.Composed = true, ""
|
||||
return next, true
|
||||
case !next.Self && next.Composed != "":
|
||||
return next, true
|
||||
}
|
||||
next.Self, next.Yes = false, ""
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
at.FP = next.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
return next, false
|
||||
}
|
||||
next.Composed = d.Digest
|
||||
return next, true
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
@@ -394,6 +565,43 @@ func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
@@ -466,12 +674,7 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
if _, barred := m.busBar(before, had, running); barred {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
@@ -483,9 +686,24 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||
prev = before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR
|
||||
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
||||
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if m.Chosen != nil {
|
||||
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
@@ -515,8 +733,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
func GrantNamesNode(g delivery.Grant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
@@ -65,13 +66,16 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
delivery.RuleCanary: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
},
|
||||
delivery.RuleProviders: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
},
|
||||
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
||||
},
|
||||
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
|
||||
delivery.RuleProvisions: func(m *Module, _ *Nodes) {
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
|
||||
}
|
||||
},
|
||||
} {
|
||||
m, n := ready()
|
||||
breakIt(&m, &n)
|
||||
@@ -85,12 +89,211 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
|
||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||
}
|
||||
if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
|
||||
t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
|
||||
for _, all := range []func(string) map[string]provisions.Provision{
|
||||
func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}}
|
||||
}, nil} {
|
||||
m, n := ready()
|
||||
m.Provisions = all
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew {
|
||||
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
|
||||
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
|
||||
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
|
||||
{
|
||||
handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}}
|
||||
keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}}
|
||||
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh,
|
||||
AnsweredBy: catalogue.AnsweredByHelper}}}
|
||||
store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}}
|
||||
goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}}
|
||||
gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}}
|
||||
answering := func(p string) []catalogue.Manifest {
|
||||
for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} {
|
||||
if m.Provides[0].Name == p {
|
||||
return []catalogue.Manifest{m}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
manifest catalogue.Manifest
|
||||
missing string // the provision named, "" for none: the want moves
|
||||
}{
|
||||
{"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"database"}}, "database"},
|
||||
{"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""},
|
||||
{"a module not handed waits for its route, which the helper answers (grafana's route)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"},
|
||||
{"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)",
|
||||
catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""},
|
||||
{"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Manifest, m.Answering = c.manifest, answering
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if c.missing == "" {
|
||||
if out.Wants["g14"].FP != fNew || len(m.MissingProvisions("g14")) != 0 {
|
||||
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out.Rules["g14"].Rule != delivery.RuleProvisions || !strings.Contains(out.Rules["g14"].Words,
|
||||
"provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 {
|
||||
t.Errorf("%s: %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
}
|
||||
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
|
||||
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
|
||||
// healthy held photos, and novox offline left a module with no canary.
|
||||
for _, c := range []struct {
|
||||
name, module, provision, provider string
|
||||
have *delivery.Have // what novox runs of the module; nil: a new assignment
|
||||
before map[string]delivery.Want
|
||||
online bool
|
||||
want string
|
||||
}{
|
||||
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
|
||||
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
|
||||
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
|
||||
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
|
||||
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
|
||||
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
|
||||
} {
|
||||
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
|
||||
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
|
||||
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
|
||||
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
|
||||
if c.have != nil {
|
||||
n.Haves["novox"][c.module] = *c.have
|
||||
}
|
||||
out := Wants(m, n, c.before, nil)
|
||||
w, written := out.Wants["novox"]
|
||||
if !written {
|
||||
w = c.before["novox"]
|
||||
}
|
||||
if w.FP != c.want || w.Removed {
|
||||
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
|
||||
}
|
||||
if _, on := out.Neither["novox"]; on {
|
||||
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
|
||||
}
|
||||
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
|
||||
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
|
||||
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
|
||||
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
|
||||
// stays and is said; a placed node's want is Self with no composition.
|
||||
selfAt := func(fp string) map[string]delivery.Want {
|
||||
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
|
||||
}
|
||||
atF := func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
set func(*Module, *Nodes)
|
||||
before string
|
||||
want delivery.Want // the want of g14 written; zero: none written
|
||||
rule int // the rule g14's holding names
|
||||
asked bool
|
||||
}{
|
||||
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
|
||||
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
|
||||
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
|
||||
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
|
||||
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
|
||||
rule: delivery.RuleNotHeld},
|
||||
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Held = true
|
||||
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
|
||||
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
|
||||
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
recording(m)
|
||||
m.Held = true
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
|
||||
}
|
||||
}, rule: delivery.RuleNotHeld, asked: true},
|
||||
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, rule: delivery.RuleComposed},
|
||||
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Self = func(node string) bool { return node == "g14" }
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, want: delivery.Want{FP: fNew, Self: true}},
|
||||
} {
|
||||
m, n := ready()
|
||||
c.set(&m, &n)
|
||||
before := selfAt(c.before)
|
||||
if c.name == "placed" {
|
||||
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
}
|
||||
out := Wants(m, n, before, nil)
|
||||
if got := out.Wants["g14"]; got != c.want {
|
||||
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
|
||||
}
|
||||
if got := out.Rules["g14"].Rule; got != c.rule {
|
||||
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
|
||||
}
|
||||
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
|
||||
t.Errorf("%s: record asks %+v", c.name, out.Record)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok {
|
||||
if _, err := delivery.EncodeWant(w); err != nil {
|
||||
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
grant := delivery.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
@@ -109,7 +312,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
if got := GrantNamesNode(delivery.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
@@ -150,6 +353,72 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
||||
}
|
||||
|
||||
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
|
||||
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
|
||||
if out.Held != nil || len(out.Wants) != 0 {
|
||||
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
|
||||
}
|
||||
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
|
||||
len(out.NodeFailed) != 1 {
|
||||
t.Errorf("node failed = %+v", out.NodeFailed)
|
||||
}
|
||||
|
||||
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
|
||||
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
|
||||
three := m
|
||||
three.Nodes = []string{"ace", "g14", "novox"}
|
||||
failedHave := n.Haves["ace"]["mail"]
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
|
||||
n.Online["novox"] = true
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
|
||||
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if _, alone := out.NodeFailed["g14"]; !alone {
|
||||
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
|
||||
}
|
||||
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
|
||||
}
|
||||
if _, written := out.Wants["g14"]; written {
|
||||
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
|
||||
}
|
||||
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
delete(before, "novox")
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
|
||||
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
|
||||
}
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
|
||||
|
||||
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
|
||||
// not one node's failure (D3). Either way F is held and ace is set back.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
online map[string]bool
|
||||
novox delivery.Health
|
||||
}{
|
||||
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
|
||||
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
|
||||
} {
|
||||
n.Online = c.online
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
|
||||
if c.novox == delivery.HealthOK {
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
|
||||
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld {
|
||||
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
||||
@@ -427,6 +696,23 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
||||
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
||||
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
||||
m.Composed = composedEverywhere
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w, wrote := out.Wants["ace"]; wrote {
|
||||
t.Errorf("the bus bar wrote ace's want %+v", w)
|
||||
}
|
||||
h, on := out.Neither["ace"]
|
||||
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
||||
!strings.Contains(h.Words, "assigner_put") {
|
||||
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
||||
}
|
||||
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
||||
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
||||
}
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
@@ -529,6 +815,13 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
|
||||
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
|
||||
m.Self = func(string) bool { return true }
|
||||
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer in a placed node's want was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
@@ -588,6 +881,12 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
||||
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
||||
}
|
||||
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
|
||||
with(delivery.PolicyRecord, "person")
|
||||
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
||||
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
@@ -789,10 +1088,12 @@ func x11() (Module, Nodes) {
|
||||
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
out := Wants(m, n, nil, nil)
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||
}
|
||||
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
|
||||
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
|
||||
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
|
||||
}
|
||||
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
|
||||
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
|
||||
}
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("left on neither list: %+v", out.Neither)
|
||||
@@ -830,11 +1131,22 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
if what == "an old-form build" {
|
||||
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("%s: shanks's want = %+v, want F", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("%s: shanks on neither list = %+v", what, h)
|
||||
}
|
||||
}
|
||||
m, n := x11()
|
||||
@@ -843,10 +1155,12 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||
}
|
||||
for node, w := range out.Wants {
|
||||
if w.FP != fOld {
|
||||
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||
}
|
||||
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
|
||||
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
|
||||
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
|
||||
}
|
||||
if w := out.Wants["g14"]; w.FP != fNew {
|
||||
t.Errorf("the canary g14's want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -857,11 +1171,12 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
m, n := x11()
|
||||
delete(n.Haves["shanks"], "screen-lock")
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
|
||||
}
|
||||
m.Built = false
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
@@ -903,7 +1218,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
oldForm := func(n Nodes, node string) {
|
||||
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
@@ -937,16 +1252,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||
}
|
||||
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
|
||||
m, n = x11()
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
|
||||
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
|
||||
}
|
||||
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||
if _, on := out.Neither["shanks"]; on {
|
||||
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
|
||||
}
|
||||
// F not granted: nothing, even with the canary healthy
|
||||
m, n = x11()
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
// Package judge is how the seat mesh-placement's holder judges a proposal (novox/hq ADR 0567 §3, §4, as ADR 0593 §8
|
||||
// changes them): what it changes on each node by its placement (Intended), every node answering it, and the verdict of
|
||||
// their answers (JudgeAnswers). The assigner's own (ADR 0593 §1), moved back from mesh-sdk's compose/judge: the holder
|
||||
// composes nothing, since every composition is its node's engine's, and a node answers through the candidates path
|
||||
// with what it composed. It holds no bus client and no timer: the holder gives it what it read.
|
||||
package judge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// AnswerBound is how long the holder waits for a node's answer after its candidate was written: the time to live of the
|
||||
// node's pulse key (delivery.PulseAnswerKey), whose expiry names it as giving no answer, never passed. The same answer
|
||||
// bound a check gives a node (ADR 0593 §8, §7: 60 seconds, delivery.ComposeAnswerBound). A variable so a test on the bus
|
||||
// can shorten it.
|
||||
var AnswerBound = delivery.ComposeAnswerBound
|
||||
|
||||
// Intended is, by node, the modules whose placed entry the proposal changes there, and the nodes that answer it: every
|
||||
// node (ADR 0593 §8, changing ADR 0567 §4's list of the nodes that must answer), sorted. current is the placement as it
|
||||
// stands, laid the proposal's placements by node (nil removes the node's key) and nodes every node with a record. A
|
||||
// node's set holds the modules the proposal adds, removes or whose placed entry differs on that node; a node with no
|
||||
// key now gets an empty set (§7: the seed is answered as the same). What a node's compositions change beyond its own
|
||||
// placement it says itself in its answer, marked in the brief as not computed.
|
||||
func Intended(current map[string]delivery.Placement, laid map[string]*delivery.Placement, nodes []string) (intended map[string]map[string]bool, must []string) {
|
||||
intended = map[string]map[string]bool{}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n] = true
|
||||
}
|
||||
for n := range laid {
|
||||
all[n] = true
|
||||
}
|
||||
for node := range all {
|
||||
must = append(must, node)
|
||||
set := map[string]bool{}
|
||||
intended[node] = set
|
||||
before, placed := current[node]
|
||||
after, changesKey := laid[node]
|
||||
if !changesKey || !placed {
|
||||
continue // untouched, or new to placement: every answer must be the same
|
||||
}
|
||||
var afterModules []delivery.PlacedModule
|
||||
if after != nil {
|
||||
afterModules = after.Modules
|
||||
}
|
||||
entries := map[string]delivery.PlacedModule{}
|
||||
for _, m := range before.Modules {
|
||||
entries[m.Module] = m
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range afterModules {
|
||||
seen[m.Module] = true
|
||||
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
|
||||
set[m.Module] = true
|
||||
}
|
||||
}
|
||||
for name := range entries {
|
||||
if !seen[name] {
|
||||
set[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(must)
|
||||
return intended, must
|
||||
}
|
||||
|
||||
// JudgeAnswers is the verdict of a proposal's answers (§4). answers is by node its HaveNode.Candidates entry for the
|
||||
// proposal (nil when it has none) and revisions by node the revision of its candidate key: an answer counts when it
|
||||
// names the proposal and that revision. It fails on a refusal by a node that must answer; a node of Must with no answer
|
||||
// that counts is waited for while its pulse key stands, and once it has ended (pulsing false: the bound's expiry, heard
|
||||
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
|
||||
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
|
||||
// shown in the ask's brief. The seed alone fails on any difference.
|
||||
func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
|
||||
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) delivery.PlacementVerdict {
|
||||
v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
|
||||
Answers: map[string]delivery.CandidateAnswer{}}
|
||||
for node, set := range intended {
|
||||
for m := range set {
|
||||
v.Intended[node] = append(v.Intended[node], m)
|
||||
}
|
||||
sort.Strings(v.Intended[node])
|
||||
}
|
||||
must := p.Must
|
||||
if len(must) == 0 {
|
||||
must = p.Nodes()
|
||||
}
|
||||
for _, node := range must {
|
||||
a := answers[node]
|
||||
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
|
||||
v.Silent = append(v.Silent, node)
|
||||
continue
|
||||
}
|
||||
v.Answers[node] = *a
|
||||
if a.Refused != "" {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s cannot compose the proposal: %s", node, a.Refused))
|
||||
continue
|
||||
}
|
||||
for _, r := range a.Refusals {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s refuses the proposal for %s: %s", node, r.Module, r.Why))
|
||||
}
|
||||
want := intended[node]
|
||||
differs := func(module, words string) {
|
||||
if p.Seed {
|
||||
// The seed carries what its nodes found, each named in the brief for the operator who approves it.
|
||||
v.Carried[node] = append(v.Carried[node], words)
|
||||
return
|
||||
}
|
||||
if !want[module] {
|
||||
v.Unintended[node] = append(v.Unintended[node], module)
|
||||
}
|
||||
}
|
||||
for _, d := range a.Differs {
|
||||
differs(d.Module, differenceWords(d))
|
||||
}
|
||||
for _, d := range a.Added {
|
||||
differs(d.Module, "would run "+d.Module)
|
||||
}
|
||||
for _, m := range a.Removed {
|
||||
differs(m, "would stop running "+m)
|
||||
}
|
||||
sort.Strings(v.Unintended[node])
|
||||
}
|
||||
if len(v.Intended) == 0 {
|
||||
v.Intended = nil
|
||||
}
|
||||
if len(v.Unintended) == 0 {
|
||||
v.Unintended = nil
|
||||
}
|
||||
if len(v.Carried) == 0 {
|
||||
v.Carried = nil
|
||||
}
|
||||
if len(v.Answers) == 0 {
|
||||
v.Answers = nil
|
||||
}
|
||||
var waiting, gone []string
|
||||
for _, node := range v.Silent {
|
||||
named := node
|
||||
if online != nil && !online[node] {
|
||||
named += " (offline)"
|
||||
}
|
||||
if pulsing[node] {
|
||||
waiting = append(waiting, named)
|
||||
} else {
|
||||
gone = append(gone, named)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("waits for the answers of %s, each for %s from its candidate's write",
|
||||
strings.Join(waiting, ", "), AnswerBound))
|
||||
}
|
||||
if len(gone) > 0 {
|
||||
// ADR 0593 §8: decided on the answers it has; that node's own composition judges again when it applies (§9).
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("no answer from %s within %s of its candidate's write: decided on the answers "+
|
||||
"it has, and that node's own composition judges again when it applies", strings.Join(gone, ", "), AnswerBound))
|
||||
}
|
||||
switch {
|
||||
case len(v.Fails) > 0:
|
||||
v.State = delivery.VerdictFail
|
||||
case len(waiting) > 0:
|
||||
v.State = delivery.VerdictWaits
|
||||
default:
|
||||
v.State = delivery.VerdictPass
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Brief is the verdict as the ask's brief shows it, by node (§5): what the proposal changes and what each node answered
|
||||
// beside it, marked, and what stays held as the node holds it today, with the proposal's why.
|
||||
func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s (proposed by %s)", p.Why, p.By)
|
||||
nodes := map[string]bool{}
|
||||
for n := range v.Intended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range v.Unintended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for _, n := range p.Nodes() {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n, a := range v.Answers {
|
||||
if len(a.Held) > 0 {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
for n := range v.Carried {
|
||||
nodes[n] = true
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, "; %s:", n)
|
||||
if m := v.Intended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, " changes %s", strings.Join(m, ", "))
|
||||
} else {
|
||||
b.WriteString(" composes the same")
|
||||
}
|
||||
if m := v.Unintended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; also differs, not computed: %s", strings.Join(m, ", "))
|
||||
}
|
||||
if m := v.Carried[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; the seed carries: %s", strings.Join(m, "; "))
|
||||
}
|
||||
if held := v.Answers[n].Held; len(held) > 0 {
|
||||
var names []string
|
||||
for _, h := range held {
|
||||
today := h.Today
|
||||
if today == "" {
|
||||
today = "held"
|
||||
}
|
||||
names = append(names, h.Module+" ("+today+")")
|
||||
}
|
||||
fmt.Fprintf(&b, "; stays as today: %s", strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
|
||||
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
|
||||
// field), and how many fields more; or the resource and why when no body is shown.
|
||||
func differenceWords(d delivery.ModuleDiffers) string {
|
||||
for _, b := range d.Bodies {
|
||||
if len(b.Paths) == 0 {
|
||||
continue
|
||||
}
|
||||
path := b.Paths[0]
|
||||
words := fmt.Sprintf("%s: %s %s → %s", d.Module, fieldName(b.Running, b.Composed, path),
|
||||
valueAt(b.Running, path), valueAt(b.Composed, path))
|
||||
if n := len(b.Paths) - 1; n > 0 {
|
||||
words += fmt.Sprintf(" (and %d more fields)", n)
|
||||
}
|
||||
return words
|
||||
}
|
||||
if d.Resource != "" {
|
||||
return fmt.Sprintf("%s: %s (%s)", d.Module, d.Resource, d.Why)
|
||||
}
|
||||
return fmt.Sprintf("%s: %s", d.Module, d.Why)
|
||||
}
|
||||
|
||||
// shownValue bounds a value named in a brief.
|
||||
const shownValue = 60
|
||||
|
||||
// walk follows a differing path (delivery.DifferingPaths: keys joined by dots, a key may hold dots, and a JSON value
|
||||
// written in a string is read as JSON) through a body, answering the keys it passed and the value at its end.
|
||||
func walk(body, path string) (keys []string, value any, ok bool) {
|
||||
var v any
|
||||
if json.Unmarshal([]byte(body), &v) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest := path
|
||||
for rest != "" {
|
||||
if s, isString := v.(string); isString {
|
||||
var inner any
|
||||
if json.Unmarshal([]byte(s), &inner) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
v = inner
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
best := ""
|
||||
for k := range t {
|
||||
if (rest == k || strings.HasPrefix(rest, k+".")) && len(k) > len(best) {
|
||||
best = k
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, best), t[best], strings.TrimPrefix(strings.TrimPrefix(rest, best), ".")
|
||||
case []any:
|
||||
seg, after, _ := strings.Cut(rest, ".")
|
||||
n, err := strconv.Atoi(seg)
|
||||
if err != nil || n < 0 || n >= len(t) {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, seg), t[n], after
|
||||
default:
|
||||
return nil, nil, false
|
||||
}
|
||||
}
|
||||
return keys, v, true
|
||||
}
|
||||
|
||||
// valueAt is the value at path in a shown body, cut to shownValue, or why it is not shown.
|
||||
func valueAt(body, path string) string {
|
||||
if body == "" {
|
||||
return "(none)"
|
||||
}
|
||||
_, v, ok := walk(body, path)
|
||||
if !ok {
|
||||
return "(value cut)"
|
||||
}
|
||||
s, isString := v.(string)
|
||||
if !isString {
|
||||
raw, _ := json.Marshal(v)
|
||||
s = string(raw)
|
||||
}
|
||||
if r := []rune(s); len(r) > shownValue {
|
||||
s = string(r[:shownValue-1]) + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fieldName is the differing field as a person reads it: its parent's name and its own, `PARENT[key]`, from whichever
|
||||
// body reaches it; the path's end when neither does.
|
||||
func fieldName(running, composed, path string) string {
|
||||
for _, body := range []string{running, composed} {
|
||||
if keys, _, ok := walk(body, path); ok && len(keys) > 0 {
|
||||
if len(keys) == 1 {
|
||||
return keys[0]
|
||||
}
|
||||
return keys[len(keys)-2] + "[" + keys[len(keys)-1] + "]"
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package judge
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §3, §4 (rows moved from the assigner's check of a pull request's placement, ADR 0559 §5): what a proposal
|
||||
// changes by node, and the verdict of the answers: differences shown and failing nothing, a refusal and a silent node
|
||||
// failing, a later answer judging again, the seed failing on any difference.
|
||||
func TestJudge(t *testing.T) {
|
||||
current := map[string]delivery.Placement{"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "server": {Node: "server"}}
|
||||
laid := map[string]*delivery.Placement{
|
||||
"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}},
|
||||
"laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: "web"}}}}
|
||||
// ADR 0593 §8: every node answers; what the placement changes is read from the placement alone.
|
||||
got, must := Intended(current, laid, []string{"anchor", "laptop", "server"})
|
||||
if !got["anchor"]["mail"] || !got["anchor"]["web"] || len(got["laptop"]) != 0 || len(got["server"]) != 0 ||
|
||||
!slices.Equal(must, []string{"anchor", "laptop", "server"}) {
|
||||
t.Errorf("intended = %v, must %v: want anchor's mail and web, nothing elsewhere, every node answering", got, must)
|
||||
}
|
||||
|
||||
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
|
||||
p := delivery.PlacementProposal{ID: "p-one", By: "op", Why: "move web", Written: at, Base: map[string]uint64{"anchor": 3, "laptop": 0},
|
||||
Must: []string{"anchor", "laptop"}}
|
||||
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
|
||||
intended := map[string]map[string]bool{"anchor": {"web": true}}
|
||||
same := func(rev uint64) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
|
||||
}
|
||||
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
seed bool
|
||||
answers map[string]*delivery.CandidateAnswer
|
||||
pulsing map[string]bool
|
||||
state string
|
||||
words string
|
||||
marked string
|
||||
}{
|
||||
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
|
||||
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
|
||||
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
|
||||
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
|
||||
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, delivery.VerdictWaits, "laptop (offline)", ""},
|
||||
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
|
||||
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
} {
|
||||
q := p
|
||||
q.Seed = row.seed
|
||||
v := JudgeAnswers(q, intended, row.answers, revisions, map[string]bool{"anchor": true}, row.pulsing, at)
|
||||
all := strings.Join(append(append([]string{}, v.Fails...), v.Notes...), "; ")
|
||||
if v.State != row.state || !strings.Contains(all, row.words) {
|
||||
t.Errorf("%s: %s, %q", row.name, v.State, all)
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass && !strings.Contains(Brief(q, v), "the seed carries: node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → (value cut) (and 1 more fields)") {
|
||||
t.Errorf("%s: the brief does not name it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass {
|
||||
whole := row.answers["laptop"].Differs[0]
|
||||
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
|
||||
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
|
||||
t.Errorf("the difference is named %q", got)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(row.name, "a module held as today") && !strings.Contains(Brief(q, v), "stays as today: spiegel (held)") {
|
||||
t.Errorf("%s: the brief does not show it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.marked != "" && (!slices.Contains(v.Unintended["anchor"], row.marked) || !strings.Contains(Brief(q, v), "not computed: "+row.marked)) {
|
||||
t.Errorf("%s: the difference is not marked: %+v, %s", row.name, v.Unintended, Brief(q, v))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func load(t *testing.T) catalogue.Manifest {
|
||||
@@ -15,9 +16,9 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
|
||||
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -32,9 +33,11 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
|
||||
// every node's published compositions to compare them.
|
||||
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
|
||||
"mesh-provisioner.provisions", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -42,7 +45,8 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -51,18 +55,38 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
// 1906/D3: it says its conditions on the operator channel; it asks the Go build seat nothing since the composition is
|
||||
// the node-engine's (ADR 0593 §7).
|
||||
if !slices.Equal(m.Uses, []string{"operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly operator-channel", m.Uses)
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||
// calls the controller's warranted and the bus node's backup holder.
|
||||
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its five verbs (adopt and hand-back went with the
|
||||
// controller, ADR 0590); keeps the seat's events on its stream placement for 30 days; calls the hand-act log; and reads
|
||||
// mesh-config's resolved values to compose every node a proposal touches.
|
||||
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
|
||||
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again"}) ||
|
||||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its five verbs and mesh-wants with its two", m.Claims)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
|
||||
}
|
||||
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
|
||||
@@ -2,28 +2,47 @@
|
||||
"module": "mesh-delivery-assigner",
|
||||
"version": "1",
|
||||
"slug": "assigner",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-placement",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"placement",
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mesh-wants",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"upgrade",
|
||||
"bus"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-grants.grants",
|
||||
"mesh-host.composed",
|
||||
"mesh-provisioner.provisions",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.now",
|
||||
"seat:node-backup.backed-up"
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:node-backup.now"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
@@ -34,11 +53,29 @@
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "placement", "history": 64 },
|
||||
{ "name": "proposals", "history": 5 },
|
||||
{ "name": "candidates", "history": 5 },
|
||||
{ "name": "pulses", "key-ttl": true },
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
{ "name": "wants", "history": 20 },
|
||||
{ "name": "findings", "history": 5 },
|
||||
{ "name": "policies", "history": 20 },
|
||||
{ "name": "seat-holders", "history": 5 }
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "placement",
|
||||
"events": [
|
||||
"placement-proposed",
|
||||
"placement-judged",
|
||||
"placement-changed",
|
||||
"placement-withdrawn"
|
||||
],
|
||||
"max-age-days": 30
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
|
||||
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
|
||||
// and the asker reads the first such key that is not running. The builder and the checker share it.
|
||||
|
||||
// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
|
||||
type AskPublisher interface {
|
||||
PublishAsk(ctx context.Context, subject, id string, data []byte) error
|
||||
}
|
||||
|
||||
// AskWaiter watches an ask's steps: every value a key `*.<ask id>` of the build module's steps state takes, running or
|
||||
// ended, on the channel it answers, until ctx ends. A watch, not a poll.
|
||||
type AskWaiter interface {
|
||||
WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error)
|
||||
}
|
||||
|
||||
// Asker puts asks and waits for their steps.
|
||||
type Asker struct {
|
||||
Publisher AskPublisher
|
||||
Waiter AskWaiter
|
||||
// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
|
||||
// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
|
||||
// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Put puts a on its stream: the seat's accept subject of its language and kind.
|
||||
func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error {
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil {
|
||||
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
|
||||
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
|
||||
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
|
||||
func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
watch, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
var bound, outerCap <-chan time.Time
|
||||
var timer, capTimer *time.Timer
|
||||
defer func() {
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
}()
|
||||
running, since, phase, phaseAt := "", time.Time{}, "", time.Time{}
|
||||
standStill := func() error {
|
||||
if phase == "" {
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound)
|
||||
}
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s",
|
||||
a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.Step{}, ctx.Err()
|
||||
case <-bound:
|
||||
return delivery.Step{}, standStill()
|
||||
case <-outerCap:
|
||||
return delivery.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended",
|
||||
a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
case step, ok := <-steps:
|
||||
if !ok {
|
||||
return delivery.Step{}, errors.New("the watch of the ask's step ended")
|
||||
}
|
||||
if seen := stepSeenOf(ctx); seen != nil {
|
||||
seen(step)
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return step, nil
|
||||
}
|
||||
if k.Bound <= 0 {
|
||||
continue
|
||||
}
|
||||
newRun := step.Node != running || !step.Started.Equal(since)
|
||||
if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) {
|
||||
continue
|
||||
}
|
||||
// A node took the ask, or took it again after its run stopped (another node, or the same node's build module
|
||||
// replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the
|
||||
// step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to
|
||||
// three bounds from its start (issue 1930).
|
||||
running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt
|
||||
last := step.Started
|
||||
if step.PhaseAt.After(last) {
|
||||
last = step.PhaseAt
|
||||
}
|
||||
left := k.Bound
|
||||
if !last.IsZero() {
|
||||
left = time.Until(last.Add(k.Bound))
|
||||
}
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
timer = time.NewTimer(max(left, 0))
|
||||
bound = timer.C
|
||||
if newRun {
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
outer := 3 * k.Bound
|
||||
if !step.Started.IsZero() {
|
||||
outer = time.Until(step.Started.Add(3 * k.Bound))
|
||||
}
|
||||
capTimer = time.NewTimer(max(outer, 0))
|
||||
outerCap = capTimer.C
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type stepSeenKey struct{}
|
||||
|
||||
// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node
|
||||
// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so
|
||||
// (issue 1890). seen runs in the waiting goroutine and must not block.
|
||||
func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context {
|
||||
return context.WithValue(ctx, stepSeenKey{}, seen)
|
||||
}
|
||||
|
||||
func stepSeenOf(ctx context.Context) func(delivery.Step) {
|
||||
seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step))
|
||||
return seen
|
||||
}
|
||||
|
||||
// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
|
||||
func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
return k.Passed(k.Wait(ctx, a))
|
||||
}
|
||||
|
||||
// Passed is step when it passed, else an error naming where it ended and why.
|
||||
func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) {
|
||||
if err != nil {
|
||||
return step, err
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
|
||||
}
|
||||
return step, nil
|
||||
}
|
||||
|
||||
// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
|
||||
// of `*.<ask id>` in the build module's steps bucket.
|
||||
type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
|
||||
msg.Header.Set(jetstream.MsgIDHeader, id)
|
||||
_, err := s.JS.PublishMsg(ctx, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
|
||||
// once, and every change after.
|
||||
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
w, err := kv.Watch(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
|
||||
}
|
||||
select {
|
||||
case out <- step:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
|
||||
// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for
|
||||
// concurrent use.
|
||||
type FakeAsks struct {
|
||||
Answer func(a delivery.StepAsk) delivery.Step
|
||||
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
steps map[string][]delivery.Step
|
||||
watchers map[string][]chan delivery.Step
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.asks = append(f.asks, a)
|
||||
f.mu.Unlock()
|
||||
if f.Answer != nil {
|
||||
f.Write(f.Answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write writes an ask's step, as a build node does.
|
||||
func (f *FakeAsks) Write(step delivery.Step) {
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
f.steps[step.ID] = append(f.steps[step.ID], step)
|
||||
ws := append([]chan delivery.Step(nil), f.watchers[step.ID]...)
|
||||
f.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- step
|
||||
}
|
||||
}
|
||||
|
||||
// End is Write: kept for the tests that end a step.
|
||||
func (f *FakeAsks) End(step delivery.Step) { f.Write(step) }
|
||||
|
||||
// WatchAsk is AskWaiter's: what was written so far, then every write.
|
||||
func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan delivery.Step, error) {
|
||||
out := make(chan delivery.Step, 64)
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
for _, s := range f.steps[askID] {
|
||||
out <- s
|
||||
}
|
||||
f.watchers[askID] = append(f.watchers[askID], out)
|
||||
f.mu.Unlock()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
f.mu.Lock()
|
||||
ws := f.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == out {
|
||||
f.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Asks are the asks put so far.
|
||||
func (f *FakeAsks) Asks() []delivery.StepAsk {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), f.asks...)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build integration
|
||||
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// 1887/A4a, 1887/AC3 on the real bus: an ask put twice under one id is one message, and the asker reads the step a node
|
||||
// wrote under `<node>.<ask id>`.
|
||||
func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Subjects: []string{"mesh.seat.node-build-go.accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket("mesh-build-go")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus := JetStreamAsks{JS: js}
|
||||
k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
|
||||
a := goAsk("mvl0frv18d1960")
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
s, _ := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
||||
if info, _ := s.Info(ctx); info.State.Msgs != 1 {
|
||||
t.Fatalf("%d messages for one ask put twice", info.State.Msgs)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
running, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
_, _ = kv.Put(ctx, key, running)
|
||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepPassed, Started: time.Now(),
|
||||
Result: json.RawMessage(`{"artifacts":[]}`)})
|
||||
_, _ = kv.Put(ctx, key, passed)
|
||||
}()
|
||||
step, err := k.Passed(k.Wait(ctx, a))
|
||||
if err != nil || step.Node != "ace" {
|
||||
t.Fatalf("step %+v, %v", step, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func goAsk(id string) delivery.StepAsk {
|
||||
return delivery.StepAsk{ID: id, Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now(),
|
||||
Build: &delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}}
|
||||
}
|
||||
|
||||
// 1887/A4a: an asker puts the ask once and answers the step a build node wrote for it, naming the node; one that did not
|
||||
// pass is an error naming the node and why; a wait of an ask already put puts nothing.
|
||||
func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
f := &FakeAsks{Answer: func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)}
|
||||
}}
|
||||
k := Asker{Publisher: f, Waiter: f}
|
||||
var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it
|
||||
step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957"))
|
||||
if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" {
|
||||
t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen)
|
||||
}
|
||||
f.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"}
|
||||
}
|
||||
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
|
||||
t.Fatalf("a refused step: %v", err)
|
||||
}
|
||||
// review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step
|
||||
// running, the step is bounded from its start, and the error names the node.
|
||||
g := &FakeAsks{}
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
|
||||
waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded {
|
||||
t.Fatalf("an ask no node took: %v, want it still waiting when ctx ended", err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1961", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1961")); err == nil || !strings.Contains(err.Error(), "on ace did not end within") {
|
||||
t.Fatalf("a step that never ends: %v", err)
|
||||
}
|
||||
if len(g.Asks()) != 0 {
|
||||
t.Fatal("a wait put an ask")
|
||||
}
|
||||
// Issue 1930: the bound is on a step that stands still, not on its age. A step that enters a new phase inside the bound is
|
||||
// waited for until it ends; one that stays in a phase the bound fails at once, naming the phase; one that never stops moving
|
||||
// ends at three bounds, said as such.
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: 60 * time.Millisecond}
|
||||
started := time.Now()
|
||||
go func() {
|
||||
for i := range 8 {
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "p" + string(rune('a'+i)), PhaseAt: time.Now()})
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepPassed})
|
||||
}()
|
||||
if step, err := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || step.State != delivery.StepPassed {
|
||||
t.Fatalf("a step that kept moving past its bound: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1981", Node: "ace", State: delivery.StepRunning, Started: time.Now(), Phase: "integration: tests", PhaseAt: time.Now()})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1981")); err == nil || !strings.Contains(err.Error(), `stood still in "integration: tests"`) {
|
||||
t.Fatalf("a step that stood still in a phase: %v", err)
|
||||
}
|
||||
loop := make(chan struct{})
|
||||
defer close(loop)
|
||||
started = time.Now()
|
||||
go func() {
|
||||
for i := 0; ; i++ {
|
||||
select {
|
||||
case <-loop:
|
||||
return
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1982", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "q" + string(rune('a'+i%26)), PhaseAt: time.Now()})
|
||||
}
|
||||
}
|
||||
}()
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1982")); err == nil || !strings.Contains(err.Error(), "kept moving") {
|
||||
t.Fatalf("a step that never stops moving: %v", err)
|
||||
}
|
||||
// Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed
|
||||
// with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written
|
||||
// first is answered as the node wrote it.
|
||||
dead := &DeadLetters{}
|
||||
lw := LostAskWaiter{Inner: g, Dead: dead}
|
||||
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
|
||||
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
|
||||
if !ok {
|
||||
t.Fatal("a dead letter's headers were not read")
|
||||
}
|
||||
go dead.Heard(given)
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970"))
|
||||
if err != nil || step.State != delivery.StepFailed || !IsLost(step) ||
|
||||
!strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") {
|
||||
t.Fatalf("a given-up ask: %+v, %v", step, err)
|
||||
}
|
||||
if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) {
|
||||
t.Fatalf("a given-up ask's error: %v", err)
|
||||
}
|
||||
step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971"))
|
||||
if err != nil || !IsNotTaken(step) || IsLost(step) {
|
||||
t.Fatalf("an ask no node took: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed})
|
||||
if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" {
|
||||
t.Fatalf("a step written: %+v, %v", step, err)
|
||||
}
|
||||
// Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and
|
||||
// tries again on the next ask waited for; heard, it says so and hands the given-up asks on.
|
||||
fol := &refusingFollower{refuse: 1}
|
||||
var mu sync.Mutex
|
||||
var told []Hearing
|
||||
hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil,
|
||||
func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() })
|
||||
hearing.Wait()
|
||||
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
|
||||
t.Fatalf("a refused hearing says %q", why)
|
||||
}
|
||||
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hearing.Wait()
|
||||
mu.Lock()
|
||||
if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 {
|
||||
t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls())
|
||||
}
|
||||
mu.Unlock()
|
||||
fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5})
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974"))
|
||||
if err != nil || !IsLost(step) {
|
||||
t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err)
|
||||
}
|
||||
// The lost check of 2026-10-28: a step whose own watch went deaf is read at the next nudge (a step of its build module
|
||||
// ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again.
|
||||
d := &FakeAsks{}
|
||||
heard := &HeardEnds{Inner: deafWatch{}, Read: d}
|
||||
k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute}
|
||||
d.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "shanks", State: delivery.StepPassed})
|
||||
got := make(chan delivery.Step, 1)
|
||||
go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }()
|
||||
for heard.Waiting() == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
heard.Nudge(NodeBuildModule)
|
||||
select {
|
||||
case s := <-got:
|
||||
t.Fatalf("a nudge of another build module answered %+v", s)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
heard.Nudge(GoBuildModule)
|
||||
if s := <-got; s.State != delivery.StepPassed || s.Node != "shanks" {
|
||||
t.Fatalf("a step its watch never handed over: %+v", s)
|
||||
}
|
||||
closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d}
|
||||
if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != delivery.StepPassed {
|
||||
t.Fatalf("a watch that closed once: %+v, %v", s, err)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
h *Hearing
|
||||
want string
|
||||
}{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} {
|
||||
var raw []byte
|
||||
if row.h != nil {
|
||||
raw, _ = json.Marshal(row.h)
|
||||
}
|
||||
if a := HearingHealth(raw); a.State != row.want || a.Why == "" {
|
||||
t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the
|
||||
// new run's start, so its pass is answered after the first run's bound would have ended the wait.
|
||||
func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
|
||||
f := &FakeAsks{}
|
||||
k := Asker{Publisher: f, Waiter: f, Bound: 60 * time.Millisecond}
|
||||
const id = "mvl0frv18d1962"
|
||||
first := time.Now()
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: first})
|
||||
go func() {
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
time.Sleep(50 * time.Millisecond) // past the first run's bound, inside the second's
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepPassed, Started: time.Now()})
|
||||
}()
|
||||
step, err := k.Wait(context.Background(), goAsk(id))
|
||||
if err != nil || step.State != delivery.StepPassed {
|
||||
t.Fatalf("step %+v, %v, want the redelivered run's pass", step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed.
|
||||
type refusingFollower struct {
|
||||
mu sync.Mutex
|
||||
refuse int
|
||||
n int
|
||||
hand func(GivenUp)
|
||||
}
|
||||
|
||||
func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.n++
|
||||
if f.n <= f.refuse {
|
||||
return nil, errors.New("permissions violation")
|
||||
}
|
||||
f.hand = heard
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n }
|
||||
|
||||
func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) }
|
||||
|
||||
// deafWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type deafWatch struct{}
|
||||
|
||||
func (deafWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
// closingWatch closes its first watch at once, and hands the later ones to inner.
|
||||
type closingWatch struct {
|
||||
inner AskWaiter
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (c *closingWatch) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if !c.closed {
|
||||
c.closed = true
|
||||
ch := make(chan delivery.Step)
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
return c.inner.WatchAsk(ctx, module, askID)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
|
||||
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
|
||||
// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
|
||||
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
|
||||
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
|
||||
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
|
||||
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
|
||||
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
|
||||
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
|
||||
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
|
||||
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
|
||||
tc, none := delivery.AgreedToolchain(entries)
|
||||
if none != "" {
|
||||
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
|
||||
}
|
||||
nodes, err := choose(language, entries, places, qualifies, want)
|
||||
return nodes, tc, err
|
||||
}
|
||||
|
||||
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
|
||||
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
|
||||
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
return choose(language, entries, places, qualifies, want)
|
||||
}
|
||||
|
||||
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
if places == nil {
|
||||
places = &Places{}
|
||||
}
|
||||
nodes := make([]string, 0, len(entries))
|
||||
var reporting []string
|
||||
seen := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if seen[e.Node] {
|
||||
continue
|
||||
}
|
||||
seen[e.Node] = true
|
||||
reporting = append(reporting, e.Node)
|
||||
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node
|
||||
nodes = append(nodes, e.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
sort.Strings(reporting)
|
||||
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
|
||||
}
|
||||
return places.choose(nodes, want), nil
|
||||
}
|
||||
|
||||
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
|
||||
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
|
||||
// building again settles nothing; it waits for another fingerprint.
|
||||
const DigestsDifferPrefix = "digests differ:"
|
||||
|
||||
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
|
||||
// digest when they differ. One node's digest agrees with itself.
|
||||
func Agree(artifact string, byNode map[string]string) (string, error) {
|
||||
nodes := make([]string, 0, len(byNode))
|
||||
for n := range byNode {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var first string
|
||||
for i, n := range nodes {
|
||||
if i == 0 {
|
||||
first = byNode[n]
|
||||
continue
|
||||
}
|
||||
if byNode[n] != first {
|
||||
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
|
||||
for _, n := range nodes {
|
||||
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
|
||||
}
|
||||
return "", fmt.Errorf("%s", msg[:len(msg)-1])
|
||||
}
|
||||
}
|
||||
return first, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
|
||||
func hex64(c byte) string { return strings.Repeat(string(c), 64) }
|
||||
|
||||
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
|
||||
var out []delivery.NodeToolchain
|
||||
for _, n := range nodes {
|
||||
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
|
||||
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
|
||||
}
|
||||
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
|
||||
t.Errorf("toolchain = %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
|
||||
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
|
||||
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
|
||||
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
|
||||
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
|
||||
}
|
||||
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
|
||||
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
|
||||
}
|
||||
}
|
||||
|
||||
func agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
|
||||
if err != nil || d != digest {
|
||||
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
|
||||
}
|
||||
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
|
||||
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
|
||||
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
|
||||
}
|
||||
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
|
||||
t.Errorf("one node: %q, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
|
||||
online := func(n string) bool { return n != "g14" } // a sleeping laptop
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
|
||||
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
|
||||
}
|
||||
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
|
||||
}
|
||||
if tc.Version != "go1.27.1" {
|
||||
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// placesRunning is Places with these steps counted as running, for a test of the choice.
|
||||
func placesRunning(running map[string]int) *Places {
|
||||
return &Places{running: running, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func choosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
|
||||
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
|
||||
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
|
||||
p.Capacity = map[string]int{"shanks": 3}
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The choice of the nodes a step runs on and the agreement of their answers, one table of cases (the test budget).
|
||||
func TestChoosingTheNodesOfAStep(t *testing.T) {
|
||||
t.Run("choose takes the agreed nodes with fewest running steps ties by name", chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName)
|
||||
t.Run("choose with one node answers one node", chooseWithOneNodeAnswersOneNode)
|
||||
t.Run("choose fails naming the language and the versions when no node qualifies", chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies)
|
||||
t.Run("agree needs equal digests and names both nodes when they differ", agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer)
|
||||
t.Run("choose skips nodes that are not online and fails naming them when none is", chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs)
|
||||
t.Run("choose prefers nodes below their capacity", choosePrefersNodesBelowTheirCapacity)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps
|
||||
// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the
|
||||
// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the
|
||||
// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484).
|
||||
package buildrun
|
||||
@@ -0,0 +1,22 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,56 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
|
||||
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
|
||||
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
|
||||
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
|
||||
// and from a build at once.
|
||||
type HaveSet struct {
|
||||
mu sync.Mutex
|
||||
healthy map[string]bool // by `<node>.<module>`
|
||||
}
|
||||
|
||||
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
|
||||
// (`<node>.node`) and a value that is not a have are left out.
|
||||
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return false
|
||||
}
|
||||
ok := false
|
||||
if !deleted {
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
||||
}
|
||||
k := node + "." + module
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.healthy[k] == ok {
|
||||
return false
|
||||
}
|
||||
if s.healthy == nil {
|
||||
s.healthy = map[string]bool{}
|
||||
}
|
||||
if ok {
|
||||
s.healthy[k] = true
|
||||
} else {
|
||||
delete(s.healthy, k)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Healthy is whether the module on the node was last seen running and healthy.
|
||||
func (s *HaveSet) Healthy(node, module string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.healthy[node+"."+module]
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A step's end is never only its own watch's to hear** (the lost check of 2026-10-28: mesh-catalog #553's verdict
|
||||
// stayed running after its step mvsgthyjed5786 had passed on shanks at 19:09:31Z; the watch of that one key, opened at
|
||||
// 18:46:05Z, heard neither the step's start nor its end and never closed, while a watch opened at 18:58:48Z heard its own
|
||||
// step end). HeardEnds stands between the asker (Asker) and the watch of each ask's step key: when it is
|
||||
// nudged, every ask it waits on reads its step keys now (a read, not the watch) and hands what it reads to its wait, so a
|
||||
// step that ended is judged even when its own watch went deaf. A watch that closes under a wait is opened again, at most
|
||||
// reopenTries times. It is nudged on an event (Nudge): any step of the same build module ending (one watch of each steps
|
||||
// state, HearEnds), a reconnect to the bus, and whatever else the asker hears (the checker: checker_ask_again).
|
||||
// It holds no timer. Every asker of the build seats waits through it (the checker and the builder, HearEnds).
|
||||
|
||||
// StepReader reads, now, every step a build module's steps state holds for an ask: its keys `*.<ask id>`.
|
||||
type StepReader interface {
|
||||
AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error)
|
||||
}
|
||||
|
||||
// reopenTries is how often a wait opens its ask's watch again after it closed, before the wait ends failed.
|
||||
const reopenTries = 3
|
||||
|
||||
// HeardEnds is an AskWaiter over Inner that also reads an ask's steps when nudged. Safe for concurrent use.
|
||||
type HeardEnds struct {
|
||||
Inner AskWaiter
|
||||
Read StepReader
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
waiting map[string]*waitedAsk // by ask id
|
||||
}
|
||||
|
||||
type waitedAsk struct {
|
||||
module string
|
||||
nudge chan struct{}
|
||||
}
|
||||
|
||||
func (h *HeardEnds) logf(format string, args ...any) {
|
||||
if h.Logf != nil {
|
||||
h.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: what Inner's watch hands, and, at each nudge, what the ask's keys hold now.
|
||||
func (h *HeardEnds) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
in, err := h.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &waitedAsk{module: module, nudge: make(chan struct{}, 1)}
|
||||
h.mu.Lock()
|
||||
if h.waiting == nil {
|
||||
h.waiting = map[string]*waitedAsk{}
|
||||
}
|
||||
h.waiting[askID] = w
|
||||
h.mu.Unlock()
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
if h.waiting[askID] == w {
|
||||
delete(h.waiting, askID)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
reopened := 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-in:
|
||||
if !ok {
|
||||
if ctx.Err() != nil || reopened >= reopenTries {
|
||||
h.logf("the watch of the ask %s's step on %s closed %d times; its wait ends", askID, module, reopened+1)
|
||||
return
|
||||
}
|
||||
reopened++
|
||||
if in, err = h.Inner.WatchAsk(ctx, module, askID); err != nil {
|
||||
h.logf("the watch of the ask %s's step on %s closed and cannot be opened again, so its wait ends: %v", askID, module, err)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the ask %s's step on %s closed; opened again (%d of %d)", askID, module, reopened, reopenTries)
|
||||
continue
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case <-w.nudge:
|
||||
if h.Read == nil {
|
||||
continue
|
||||
}
|
||||
steps, err := h.Read.AskSteps(ctx, module, askID)
|
||||
if err != nil {
|
||||
h.logf("the steps of the ask %s on %s cannot be read: %v", askID, module, err)
|
||||
continue
|
||||
}
|
||||
for _, s := range steps {
|
||||
if s.State != delivery.StepRunning {
|
||||
h.logf("the ask %s's step on %s ended %s, read at a nudge: its watch had not handed it over", askID, s.Node, s.State)
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Nudge has every ask waited on of module read its steps now; module "" nudges every ask.
|
||||
func (h *HeardEnds) Nudge(module string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for _, w := range h.waiting {
|
||||
if module == "" || w.module == module {
|
||||
select {
|
||||
case w.nudge <- struct{}{}:
|
||||
default: // a nudge is pending already
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Waiting is how many asks are waited on now.
|
||||
func (h *HeardEnds) Waiting() int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return len(h.waiting)
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the keys `*.<ask id>` of the build module's steps state, read now.
|
||||
func (s JetStreamAsks) AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
lister, err := kv.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []delivery.Step
|
||||
for key := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue // deleted between the list and the read
|
||||
}
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
out = append(out, step)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HearEnds is the HeardEnds every asker of the build seats waits through: over inner (the asks' own watches, or a
|
||||
// LostAskWaiter's inner), reading with asks, and nudged at each step end of each of modules (one watch of each steps state,
|
||||
// updates only; one that closes is opened again, and said and left after reopenTries closes in a row that heard nothing). The asker nudges it at a reconnect too (Nudge("")).
|
||||
func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules []string, logf func(string, ...any)) *HeardEnds {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf}
|
||||
for _, module := range modules {
|
||||
kv, err := asks.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
|
||||
continue
|
||||
}
|
||||
go h.follow(ctx, kv, module)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) {
|
||||
silent := 0 // closes in a row having heard nothing
|
||||
for ctx.Err() == nil {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
h.logf("the steps of %s cannot be watched, so an ask whose own watch goes deaf is read only at a reconnect: %v", module, err)
|
||||
return
|
||||
}
|
||||
heard := false
|
||||
for e := range w.Updates() {
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
heard = true
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil && step.State != delivery.StepRunning {
|
||||
h.Nudge(module)
|
||||
}
|
||||
}
|
||||
_ = w.Stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
h.Nudge(module) // a step may have ended while the watch was closing
|
||||
if heard {
|
||||
silent = 0
|
||||
} else if silent++; silent >= reopenTries {
|
||||
h.logf("the watch of the steps of %s closed %d times in a row having heard nothing, so it is left: an ask whose own watch goes deaf is read only at a reconnect", module, silent)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the steps of %s closed; opened again", module)
|
||||
}
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the last step written for the ask, if any.
|
||||
func (f *FakeAsks) AskSteps(_ context.Context, _ string, askID string) ([]delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if s := f.steps[askID]; len(s) > 0 {
|
||||
return []delivery.Step{s[len(s)-1]}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask
|
||||
// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node
|
||||
// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under
|
||||
// mesh.events.dead.<seat stream>.<worker>, with the ask's message id (its ask id), how often it was handed out and when it
|
||||
// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays
|
||||
// running and holds a try place, a check's verdict stays running and its pull request waits.
|
||||
//
|
||||
// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on
|
||||
// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose
|
||||
// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its
|
||||
// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the
|
||||
// builder asks its part of a build again).
|
||||
//
|
||||
// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took
|
||||
// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise
|
||||
// (ADR 0525 §9): a backlog is no fault.
|
||||
|
||||
// LostNode is the node a step ended for a lost ask names: no build node wrote it.
|
||||
const LostNode = "bus"
|
||||
|
||||
// The words of a step ended for its ask.
|
||||
const (
|
||||
LostWords = "its ask was given up by the build seat"
|
||||
NotTakenWords = "no build node took its ask"
|
||||
)
|
||||
|
||||
// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link
|
||||
// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up.
|
||||
const (
|
||||
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
|
||||
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
|
||||
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
|
||||
DeadHeaderStream = "Mesh-Dead-Stream"
|
||||
)
|
||||
|
||||
// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat
|
||||
// stream's copies there (buscompose DeadLetterSubject).
|
||||
const DeadLettersStream = "DEAD_LETTERS"
|
||||
|
||||
// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under.
|
||||
func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" }
|
||||
|
||||
// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-<language>).
|
||||
func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) }
|
||||
|
||||
// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it.
|
||||
type GivenUp struct {
|
||||
AskID string
|
||||
Stream string
|
||||
Deliveries int
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id.
|
||||
func GivenUpOf(header map[string][]string) (GivenUp, bool) {
|
||||
get := func(k string) string {
|
||||
if v := header[k]; len(v) > 0 {
|
||||
return strings.TrimSpace(v[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)}
|
||||
if g.AskID == "" {
|
||||
return GivenUp{}, false
|
||||
}
|
||||
g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries))
|
||||
g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp))
|
||||
return g, true
|
||||
}
|
||||
|
||||
// Words are what a step ended for the given-up ask says.
|
||||
func (g GivenUp) Words() string {
|
||||
at := "at an unknown time"
|
||||
if !g.At.IsZero() {
|
||||
at = "at " + g.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream)
|
||||
}
|
||||
|
||||
// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard.
|
||||
const keptGivenUp = 1024
|
||||
|
||||
// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use.
|
||||
type DeadLetters struct {
|
||||
mu sync.Mutex
|
||||
heard map[string]GivenUp
|
||||
order []string
|
||||
watchers map[string][]chan GivenUp
|
||||
}
|
||||
|
||||
// Heard takes one given-up ask in.
|
||||
func (d *DeadLetters) Heard(g GivenUp) {
|
||||
d.mu.Lock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if _, ok := d.heard[g.AskID]; !ok {
|
||||
d.order = append(d.order, g.AskID)
|
||||
if len(d.order) > keptGivenUp {
|
||||
delete(d.heard, d.order[0])
|
||||
d.order = d.order[1:]
|
||||
}
|
||||
}
|
||||
d.heard[g.AskID] = g
|
||||
ws := d.watchers[g.AskID]
|
||||
delete(d.watchers, g.AskID)
|
||||
d.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- g // buffered, one each
|
||||
}
|
||||
}
|
||||
|
||||
// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel.
|
||||
func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) {
|
||||
ch := make(chan GivenUp, 1)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if g, ok := d.heard[askID]; ok {
|
||||
ch <- g
|
||||
return ch, func() {}
|
||||
}
|
||||
d.watchers[askID] = append(d.watchers[askID], ch)
|
||||
return ch, func() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
ws := d.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == ch {
|
||||
d.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(d.watchers[askID]) == 0 {
|
||||
delete(d.watchers, askID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type notTakenKey struct{}
|
||||
|
||||
// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's).
|
||||
func NotTakenBound(ctx context.Context, bound time.Duration) context.Context {
|
||||
return context.WithValue(ctx, notTakenKey{}, bound)
|
||||
}
|
||||
|
||||
func notTakenBoundOf(ctx context.Context) time.Duration {
|
||||
d, _ := ctx.Value(notTakenKey{}).(time.Duration)
|
||||
return d
|
||||
}
|
||||
|
||||
// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under
|
||||
// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on
|
||||
// the node "bus". Every step the build module writes is handed on as it is.
|
||||
type LostAskWaiter struct {
|
||||
Inner AskWaiter
|
||||
Dead *DeadLetters
|
||||
// Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is
|
||||
// the event that makes hearing matter.
|
||||
Hearing interface {
|
||||
Deaf() string
|
||||
Again()
|
||||
}
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (w LostAskWaiter) now() time.Time {
|
||||
if w.Now != nil {
|
||||
return w.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's.
|
||||
func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if w.Hearing != nil && w.Hearing.Deaf() != "" {
|
||||
w.Hearing.Again()
|
||||
}
|
||||
inner, err := w.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dead <-chan GivenUp
|
||||
cancel := func() {}
|
||||
if w.Dead != nil {
|
||||
dead, cancel = w.Dead.watch(askID)
|
||||
}
|
||||
// The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask.
|
||||
var notTaken <-chan struct{}
|
||||
bound := notTakenBoundOf(ctx)
|
||||
stopBound := func() {}
|
||||
if bound > 0 {
|
||||
var nt context.Context
|
||||
nt, stopBound = context.WithTimeout(context.Background(), bound)
|
||||
notTaken = nt.Done()
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer cancel()
|
||||
defer func() { stopBound() }()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
ended := func(reason string) delivery.Step {
|
||||
at := w.now().UTC()
|
||||
return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-inner:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if notTaken != nil {
|
||||
stopBound() // a node took it: its step is bounded by the asker from its start
|
||||
notTaken = nil
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case g := <-dead:
|
||||
dead = nil
|
||||
send(ended(g.Words()))
|
||||
return
|
||||
case <-notTaken:
|
||||
send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound)))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords).
|
||||
func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) }
|
||||
|
||||
// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound.
|
||||
func IsNotTaken(s delivery.Step) bool {
|
||||
return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords)
|
||||
}
|
||||
|
||||
// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is
|
||||
// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is
|
||||
// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in
|
||||
// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every
|
||||
// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so
|
||||
// the module's health tool says it: a deaf asker is not healthy.
|
||||
|
||||
// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not.
|
||||
type Hearing struct {
|
||||
Heard bool `json:"heard"`
|
||||
Streams []string `json:"streams"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing).
|
||||
const HearingKey = "dead-letters"
|
||||
|
||||
// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to
|
||||
// heard; it answers the stop, or why it cannot.
|
||||
type DeadLetterFollower interface {
|
||||
Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error)
|
||||
}
|
||||
|
||||
// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the
|
||||
// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again.
|
||||
type JetStreamDeadLetters struct{ JS jetstream.JetStream }
|
||||
|
||||
// Follow is DeadLetterFollower's.
|
||||
func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) {
|
||||
cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
if g, ok := GivenUpOf(m.Headers()); ok {
|
||||
heard(g)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cc.Stop, nil
|
||||
}
|
||||
|
||||
// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing.
|
||||
type DeadLetterHearing struct {
|
||||
follower DeadLetterFollower
|
||||
streams []string
|
||||
dead *DeadLetters
|
||||
logf func(string, ...any)
|
||||
said func(Hearing)
|
||||
ctx context.Context
|
||||
since time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
stops map[string]func()
|
||||
why map[string]string
|
||||
trying bool
|
||||
again bool
|
||||
told *Hearing
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said
|
||||
// is told each change of whether it hears (nil tells nothing); ctx ends it.
|
||||
func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters,
|
||||
logf func(string, ...any), said func(Hearing)) *DeadLetterHearing {
|
||||
h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(),
|
||||
stops: map[string]func(){}, why: map[string]string{}}
|
||||
for _, l := range languages {
|
||||
s := SeatStream(l)
|
||||
h.streams = append(h.streams, s)
|
||||
h.why[s] = "not tried yet"
|
||||
}
|
||||
if h.logf == nil {
|
||||
h.logf = func(string, ...any) {}
|
||||
}
|
||||
h.Again()
|
||||
return h
|
||||
}
|
||||
|
||||
// Deaf says why a stream is not heard, or "" when every one is.
|
||||
func (h *DeadLetterHearing) Deaf() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.deafLocked()
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) deafLocked() string {
|
||||
var out []string
|
||||
for _, s := range h.streams {
|
||||
if why, deaf := h.why[s]; deaf {
|
||||
out = append(out, s+": "+why)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " +
|
||||
strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after.
|
||||
func (h *DeadLetterHearing) Again() {
|
||||
h.mu.Lock()
|
||||
if h.trying {
|
||||
h.again = true
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.trying = true
|
||||
h.mu.Unlock()
|
||||
h.wg.Add(1)
|
||||
go func() {
|
||||
defer h.wg.Done()
|
||||
for {
|
||||
h.try()
|
||||
h.mu.Lock()
|
||||
if !h.again || h.ctx.Err() != nil {
|
||||
h.trying = false
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.again = false
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait returns when no try runs.
|
||||
func (h *DeadLetterHearing) Wait() { h.wg.Wait() }
|
||||
|
||||
// Stop ends every stream's following.
|
||||
func (h *DeadLetterHearing) Stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for s, stop := range h.stops {
|
||||
stop()
|
||||
delete(h.stops, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) try() {
|
||||
for _, s := range h.streams {
|
||||
h.mu.Lock()
|
||||
_, deaf := h.why[s]
|
||||
h.mu.Unlock()
|
||||
if !deaf || h.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
stream := s
|
||||
stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) {
|
||||
h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream,
|
||||
g.Deliveries, g.At.Format(time.RFC3339))
|
||||
h.dead.Heard(g)
|
||||
})
|
||||
h.mu.Lock()
|
||||
if err != nil {
|
||||
h.why[stream] = err.Error()
|
||||
h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v",
|
||||
stream, DeadLetterFilter(stream), DeadLettersStream, err)
|
||||
} else {
|
||||
delete(h.why, stream)
|
||||
h.stops[stream] = stop
|
||||
h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
h.tell()
|
||||
}
|
||||
|
||||
// tell says the hearing when it changed.
|
||||
func (h *DeadLetterHearing) tell() {
|
||||
h.mu.Lock()
|
||||
why := h.deafLocked()
|
||||
now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()}
|
||||
same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why
|
||||
if !same {
|
||||
h.told = &now
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !same && h.said != nil {
|
||||
h.said(now)
|
||||
}
|
||||
}
|
||||
|
||||
// HearingState is the name of the state an asker keeps its Hearing in (`<module>_hearing`, key HearingKey), which its
|
||||
// health tool reads.
|
||||
const HearingState = "hearing"
|
||||
|
||||
// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be
|
||||
// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet.
|
||||
type HearingKeeper interface {
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and
|
||||
// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits.
|
||||
func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) {
|
||||
if kv == nil {
|
||||
if logf != nil {
|
||||
logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return func(h Hearing) {
|
||||
data, err := json.Marshal(h)
|
||||
if err == nil {
|
||||
_, err = kv.Put(ctx, HearingKey, data)
|
||||
}
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil {
|
||||
logf("whether %s is heard could not be kept: %v", DeadLettersStream, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears
|
||||
// DEAD_LETTERS, unhealthy saying why while it does not or has not said.
|
||||
func HearingHealth(raw []byte) health.Answer {
|
||||
if len(raw) == 0 {
|
||||
return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there")
|
||||
}
|
||||
var h Hearing
|
||||
if err := json.Unmarshal(raw, &h); err != nil {
|
||||
return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error())
|
||||
}
|
||||
if !h.Heard {
|
||||
return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)")
|
||||
}
|
||||
return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", "))
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package buildrun
|
||||
|
||||
// The build modules the mesh runs, by the language each builds (ADR 0469).
|
||||
const (
|
||||
GoBuildModule = "mesh-build-go"
|
||||
NodeBuildModule = "mesh-build-node"
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
|
||||
const DefaultCapacity = 1
|
||||
|
||||
// Places counts the build steps a part runs on each node and holds a lane back while its node is at its
|
||||
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
|
||||
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
|
||||
// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's
|
||||
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
|
||||
// other lanes, not by a timer: ending a lane wakes the waiting ones.
|
||||
type Places struct {
|
||||
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
|
||||
Capacity map[string]int
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]int // lanes holding a place
|
||||
reserved map[string]int // lanes chosen for the node that do not hold a place yet
|
||||
changed chan struct{} // closed, and replaced, each time a lane ends
|
||||
}
|
||||
|
||||
// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not
|
||||
// name, and 0 for one it names so, where the part never builds.
|
||||
func (p *Places) CapacityOf(node string) int {
|
||||
if n, ok := p.Capacity[node]; ok && n >= 0 {
|
||||
return n
|
||||
}
|
||||
return DefaultCapacity
|
||||
}
|
||||
|
||||
// Running is the steps in flight now, by node.
|
||||
func (p *Places) Running() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.running))
|
||||
for n, c := range p.running {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Reserved is the lanes chosen for a node that hold no place yet, by node.
|
||||
func (p *Places) Reserved() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.reserved))
|
||||
for n, c := range p.reserved {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Places) init() {
|
||||
if p.running == nil {
|
||||
p.running = map[string]int{}
|
||||
}
|
||||
if p.reserved == nil {
|
||||
p.reserved = map[string]int{}
|
||||
}
|
||||
if p.changed == nil {
|
||||
p.changed = make(chan struct{})
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
|
||||
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
|
||||
// gives up the reservation too.
|
||||
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
|
||||
for {
|
||||
p.mu.Lock()
|
||||
p.init()
|
||||
if p.running[node] < p.CapacityOf(node) {
|
||||
p.running[node]++
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
p.mu.Lock()
|
||||
p.running[node]--
|
||||
close(p.changed)
|
||||
p.changed = make(chan struct{})
|
||||
p.mu.Unlock()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
wait := p.changed
|
||||
p.mu.Unlock()
|
||||
select {
|
||||
case <-wait:
|
||||
case <-ctx.Done():
|
||||
p.Cancel(node)
|
||||
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel gives up a place reserved for the node that will not be taken.
|
||||
func (p *Places) Cancel(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
}
|
||||
|
||||
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
|
||||
// them, then by name), keeps the first want and reserves a place on each, all at once.
|
||||
func (p *Places) choose(nodes []string, want int) []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
load := func(n string) int { return p.running[n] + p.reserved[n] }
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
|
||||
if ri != rj {
|
||||
return ri
|
||||
}
|
||||
if load(nodes[i]) != load(nodes[j]) {
|
||||
return load(nodes[i]) < load(nodes[j])
|
||||
}
|
||||
return nodes[i] < nodes[j]
|
||||
})
|
||||
if len(nodes) > want {
|
||||
nodes = nodes[:want]
|
||||
}
|
||||
for _, n := range nodes {
|
||||
p.reserved[n]++
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func acquireAsync(p *Places, node string) chan func() {
|
||||
got := make(chan func(), 1)
|
||||
go func() {
|
||||
release, err := p.Acquire(context.Background(), node)
|
||||
if err != nil {
|
||||
close(got)
|
||||
return
|
||||
}
|
||||
got <- release
|
||||
}()
|
||||
return got
|
||||
}
|
||||
|
||||
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"ace": 2}}
|
||||
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
|
||||
ra, rb := <-a, <-b
|
||||
third := acquireAsync(p, "ace")
|
||||
select {
|
||||
case <-third:
|
||||
t.Fatal("a third step started on a node of capacity 2 with two running")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if p.Running()["ace"] != 2 {
|
||||
t.Errorf("running = %v, want 2 on ace", p.Running())
|
||||
}
|
||||
ra()
|
||||
select {
|
||||
case r := <-third:
|
||||
r()
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the third step did not start when one ended")
|
||||
}
|
||||
rb()
|
||||
}
|
||||
|
||||
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3}}
|
||||
first := <-acquireAsync(p, "g14")
|
||||
second := acquireAsync(p, "g14")
|
||||
select {
|
||||
case <-second:
|
||||
t.Fatal("two steps on a node without a setting")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
first()
|
||||
r := <-second
|
||||
r()
|
||||
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
|
||||
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
|
||||
p := &Places{}
|
||||
r := <-acquireAsync(p, "ace")
|
||||
defer r()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Error("Acquire returned no error for a cancelled wait")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a cancelled wait did not return")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
|
||||
func TestACapacityOfZeroKeepsThePartOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3, "ace": 2, "g14": 0, "novox": 0}}
|
||||
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
|
||||
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
|
||||
}
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
|
||||
for i := 0; i < 6; i++ {
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n == "g14" || n == "novox" {
|
||||
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
|
||||
_, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "capacity") {
|
||||
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
|
||||
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
|
||||
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
|
||||
type PresenceSet struct {
|
||||
mu sync.Mutex
|
||||
online map[string]bool
|
||||
}
|
||||
|
||||
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
|
||||
// value that is not a presence are left out.
|
||||
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, part, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || part != "" {
|
||||
return false
|
||||
}
|
||||
on := false
|
||||
if !deleted {
|
||||
p, err := delivery.DecodePresence(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
on = p.Online
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.online[node] == on {
|
||||
return false
|
||||
}
|
||||
if s.online == nil {
|
||||
s.online = map[string]bool{}
|
||||
}
|
||||
if on {
|
||||
s.online[node] = true
|
||||
} else {
|
||||
delete(s.online, node)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Online is whether the watchman last saw the node online.
|
||||
func (s *PresenceSet) Online(node string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.online[node]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func presenceValue(t *testing.T, online bool) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
|
||||
var s PresenceSet
|
||||
if s.Online("ace") {
|
||||
t.Error("a node the watchman never wrote is online")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
|
||||
t.Error("ace online was not taken in")
|
||||
}
|
||||
if s.Put("ace", presenceValue(t, true), false) {
|
||||
t.Error("the same presence again counted as a change")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
|
||||
t.Error("ace going offline was not taken in")
|
||||
}
|
||||
s.Put("g14", presenceValue(t, true), false)
|
||||
if !s.Put("g14", nil, true) || s.Online("g14") {
|
||||
t.Error("a deleted key is still online")
|
||||
}
|
||||
// A part's instance (part.<part>.<instance>) is not a node.
|
||||
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
|
||||
t.Error("a part's key was taken for a node")
|
||||
}
|
||||
if s.Put("novox", []byte("junk"), false) {
|
||||
t.Error("junk counted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the
|
||||
// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left
|
||||
// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// ToolchainSet): an entry is passed on only while the placement (the seat mesh-placement's state `placement`, held by
|
||||
// the assigner, delivery.PlacementBucket, read by delivery.Placed; ADR 0567, ADR 0549 §3) places its build module on
|
||||
// its node, and only once it was written since that placement; it is passed on as deleted otherwise, said once, and
|
||||
// passed on when its node writes it again. A report written before the placement is one an earlier run of the module
|
||||
// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23
|
||||
// back into the agreement before the module ran there). Until the placement is settled, and while no node's placement
|
||||
// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed
|
||||
// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465,
|
||||
// and it stays.
|
||||
//
|
||||
// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's
|
||||
// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6).
|
||||
type ToolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
placed *delivery.Placed
|
||||
known bool // the placement is settled
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what
|
||||
// it leaves out.
|
||||
func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate {
|
||||
g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: delivery.NewPlaced(),
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Placement takes one revision of the seat mesh-placement's placement in (at is when the bus wrote it) and answers whether what it
|
||||
// passed on changed. A value that does not read is said, and places nothing on its node.
|
||||
func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was := g.placed.Seeded()
|
||||
node, changed, err := g.placed.Put(key, value, deleted, at)
|
||||
if err != nil {
|
||||
g.logf("%s: %v (issue 1940)", delivery.PlacementBucket, err)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
if was != g.placed.Seeded() {
|
||||
// The first placement came, or the last went: every report's standing changed.
|
||||
return g.all()
|
||||
}
|
||||
return g.again(node, changed)
|
||||
}
|
||||
|
||||
// Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's
|
||||
// placement is held. It answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
return g.all()
|
||||
}
|
||||
|
||||
// all passes on every report again. The caller holds mu.
|
||||
func (g *ToolchainGate) all() bool {
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// again passes on every report of node by a build module among modules. The caller holds mu.
|
||||
func (g *ToolchainGate) again(node string, modules []string) bool {
|
||||
changed := false
|
||||
for _, module := range modules {
|
||||
if !g.modules[module] {
|
||||
continue
|
||||
}
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the placement is not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *ToolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known && g.placed.Seeded() {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its
|
||||
// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *ToolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, placed := g.placed.Since(node, module)
|
||||
if !placed {
|
||||
return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FollowPlacement reads the seat mesh-placement's placement with its history into gate, says it settled, and follows it; changed
|
||||
// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part
|
||||
// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that
|
||||
// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel.
|
||||
func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, delivery.PlacementBucket)
|
||||
if err != nil {
|
||||
logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
put := func(e jetstream.KeyValueEntry) bool {
|
||||
return gate.Placement(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut, e.Created())
|
||||
}
|
||||
for settled := false; !settled; {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = w.Stop()
|
||||
return errs, ctx.Err()
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errs, errors.New("the watch of " + delivery.PlacementBucket + " ended before it settled")
|
||||
}
|
||||
if e == nil {
|
||||
settled = true
|
||||
continue
|
||||
}
|
||||
put(e)
|
||||
}
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs <- ctx.Err()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
errs <- errors.New("the watch of " + delivery.PlacementBucket + " ended")
|
||||
return
|
||||
}
|
||||
if e != nil && put(e) {
|
||||
changed(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
// placing is a value of the state `placement` placing modules on node.
|
||||
func placing(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
return key, values[key]
|
||||
}
|
||||
|
||||
// placingLegacy is placing in the form before, `placement.<node>` (research 072 row 37).
|
||||
func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
key, _ := delivery.LegacyPlacementKey(node)
|
||||
value, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while
|
||||
// the placement places its build module on its node, and only once written since; before the placement is settled, and
|
||||
// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left
|
||||
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
|
||||
// back before the module ran there.
|
||||
func TestToolchainGate(t *testing.T) {
|
||||
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
|
||||
three := []string{"g14", "novox", "shanks"}
|
||||
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
|
||||
set := &ToolchainSet{}
|
||||
said := &[]string{}
|
||||
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range three {
|
||||
if seeded {
|
||||
key, value := placing(t, node, NodeBuildModule)
|
||||
gate.Placement(key, value, false, old.Add(-time.Hour))
|
||||
}
|
||||
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
}
|
||||
return gate, set, said
|
||||
}
|
||||
|
||||
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
|
||||
t.Fatalf("before the placement is settled every report counts; got %+v", a)
|
||||
}
|
||||
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
|
||||
gate.Placement(k, v, false, old)
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
|
||||
t.Fatalf("ace's report is said left out once; said %q", *said)
|
||||
}
|
||||
if gate.Toolchain(NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node not placed changed what was passed on")
|
||||
}
|
||||
// Placed on ace with a report written since: it counts, and the builders really disagree.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
gate.Placement(k, v, false, old.Add(-time.Minute))
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
|
||||
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
|
||||
}
|
||||
// Taken off ace again (its key deleted): its report goes again.
|
||||
if !gate.Placement(k, nil, true, placed) {
|
||||
t.Fatal("taking ace's placement away passed nothing on")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is placed or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1", placed)
|
||||
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
|
||||
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
|
||||
}
|
||||
})
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"a report left before the placement", old, false},
|
||||
{"a report written since the placement", placed.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
k, v := placing(t, "ace") // ace's key, first heard placing nothing
|
||||
gate.Placement(k, v, false, old.Add(-time.Hour))
|
||||
gate.Settled()
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed) {
|
||||
t.Fatal("a placement with no report passed something on")
|
||||
}
|
||||
key, value := gateReport(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// ace's placement written again with another module beside: its standing does not move, nothing is said again.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed.Add(time.Minute)) {
|
||||
t.Fatal("the same placement of mesh-build-node changed what was passed on")
|
||||
}
|
||||
if c.counts {
|
||||
return
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", *said)
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
|
||||
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("no placement held: every report counts", func(t *testing.T) {
|
||||
gate, set, _ := start(t, false)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
gate.Settled()
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("before the first placement file every report counts; got %d", n)
|
||||
}
|
||||
k, v := placing(t, "g14", NodeBuildModule)
|
||||
if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
|
||||
t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript")))
|
||||
}
|
||||
if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("the last placement gone: every report counts again")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
|
||||
p := delivery.NewPlaced()
|
||||
// ace in the form before, then converted: its module stands since before, the legacy key read no more.
|
||||
k, v := placingLegacy(t, "ace", NodeBuildModule)
|
||||
p.Put(k, v, false, old)
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil {
|
||||
t.Errorf("the conversion of ace's key changed %v, %v", changed, err)
|
||||
}
|
||||
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
|
||||
t.Errorf("a module key says which modules stand: %v", changed)
|
||||
}
|
||||
if k, v := placingLegacy(t, "ace"); true {
|
||||
if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 {
|
||||
t.Errorf("the legacy key is read after the node key: %v", changed)
|
||||
}
|
||||
}
|
||||
k2, v2 := placing(t, "g14")
|
||||
p.Put(k2, v2, false, old)
|
||||
k2, v2 = placing(t, "g14", NodeBuildModule)
|
||||
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
|
||||
t.Errorf("a module placed says changed %v", changed)
|
||||
}
|
||||
if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil {
|
||||
t.Error("a value that does not read was not said")
|
||||
}
|
||||
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
|
||||
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
|
||||
}
|
||||
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
|
||||
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
|
||||
}
|
||||
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
|
||||
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement")
|
||||
gate, _, _ := start(t, false)
|
||||
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
|
||||
if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
|
||||
t.Errorf("a refused placement stopped the start: %v", err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
|
||||
type Caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
|
||||
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
|
||||
// ends or ctx does.
|
||||
type Steps interface {
|
||||
Wait(ctx context.Context, module, key string) (delivery.Step, error)
|
||||
}
|
||||
|
||||
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
|
||||
// key when the step ends (ADR 0469 clause 2).
|
||||
type Runner struct {
|
||||
Caller Caller
|
||||
Steps Steps
|
||||
// Bound is how long a step may run before the builder stops waiting for it.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
|
||||
// failed or was refused is an error naming the node, the state and the reason.
|
||||
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
|
||||
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
var answer struct {
|
||||
Step string `json:"step"`
|
||||
Key string `json:"key"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
|
||||
}
|
||||
if answer.Key == "" {
|
||||
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
|
||||
}
|
||||
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
|
||||
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
|
||||
}
|
||||
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
|
||||
defer cancel()
|
||||
step, err := r.Steps.Wait(wait, module, answer.Key)
|
||||
if err != nil {
|
||||
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
|
||||
err = cause
|
||||
}
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
|
||||
}
|
||||
return step.Result, nil
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
type call struct {
|
||||
node, key string
|
||||
body any
|
||||
}
|
||||
|
||||
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
|
||||
type fakeCaller struct {
|
||||
calls []call
|
||||
answer string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
f.calls = append(f.calls, call{node, key, body})
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return json.RawMessage(f.answer), nil
|
||||
}
|
||||
|
||||
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
|
||||
type fakeSteps struct {
|
||||
mu sync.Mutex
|
||||
steps map[string]delivery.Step
|
||||
// during, when set, is called as a wait starts and the function it answers as the wait ends.
|
||||
during func(key string) func()
|
||||
}
|
||||
|
||||
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
s, ok := f.steps[module+"/"+key]
|
||||
during := f.during
|
||||
f.mu.Unlock()
|
||||
if ok {
|
||||
if during != nil {
|
||||
defer during(key)()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return delivery.Step{}, context.Cause(ctx)
|
||||
}
|
||||
|
||||
func stepAt(node, id, state, reason string, result string) delivery.Step {
|
||||
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
|
||||
if result != "" {
|
||||
s.Result = json.RawMessage(result)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func runCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
|
||||
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
|
||||
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
|
||||
}
|
||||
var fr struct{ Workspace string }
|
||||
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
|
||||
t.Errorf("result = %s (%v), want the step's result", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func runFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
|
||||
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
|
||||
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
|
||||
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
|
||||
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
|
||||
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if err == nil {
|
||||
t.Errorf("answer %s: no error", answer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
|
||||
boom := errors.New("no responders")
|
||||
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("error = %v, want the call's error naming the node", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A step run on its node and read from its steps key, one table of cases (the test budget).
|
||||
func TestRunningAStepOnANode(t *testing.T) {
|
||||
t.Run("run calls the tool on the node and reads the steps key", runCallsTheToolOnTheNodeAndReadsTheStepsKey)
|
||||
t.Run("run fails on a failed or refused step naming its reason", runFailsOnAFailedOrRefusedStepNamingItsReason)
|
||||
t.Run("run stops waiting at the bound", runStopsWaitingAtTheBound)
|
||||
t.Run("run refuses an answer with no key and a key of another node", runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode)
|
||||
t.Run("run returns the calls own error", runReturnsTheCallsOwnError)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
|
||||
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
|
||||
|
||||
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
|
||||
// put them in. Safe for use from the watches and from a build at once.
|
||||
type ToolchainSet struct {
|
||||
mu sync.Mutex
|
||||
by map[string]map[string]delivery.Toolchain // language -> node -> entry
|
||||
}
|
||||
|
||||
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
|
||||
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
|
||||
// A key or a value that is not an entry of the module's language is left out.
|
||||
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
|
||||
language, ok := LanguageOf[module]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
node, keyLanguage, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || keyLanguage != language {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := s.by[language][node]; !had {
|
||||
return false
|
||||
}
|
||||
delete(s.by[language], node)
|
||||
return true
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err != nil || tc.Language != language {
|
||||
return false
|
||||
}
|
||||
if s.by == nil {
|
||||
s.by = map[string]map[string]delivery.Toolchain{}
|
||||
}
|
||||
if s.by[language] == nil {
|
||||
s.by[language] = map[string]delivery.Toolchain{}
|
||||
}
|
||||
cur, had := s.by[language][node]
|
||||
s.by[language][node] = tc
|
||||
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
|
||||
}
|
||||
|
||||
// Entries are the nodes' entries of a language, in no particular order.
|
||||
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []delivery.NodeToolchain
|
||||
for node, tc := range s.by[language] {
|
||||
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func tcValue(t *testing.T, language, version string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
|
||||
var s ToolchainSet
|
||||
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("a first entry did not count as a change")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("the same entry again counted as a change")
|
||||
}
|
||||
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
|
||||
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
if got := s.Entries("go"); len(got) != 2 {
|
||||
t.Errorf("go entries = %+v, want two", got)
|
||||
}
|
||||
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
|
||||
t.Errorf("typescript entries = %+v", got)
|
||||
}
|
||||
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
|
||||
t.Error("a delete did not remove the node's entry")
|
||||
}
|
||||
// What is not a toolchain of the module's language is left out.
|
||||
s = ToolchainSet{}
|
||||
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
|
||||
t.Error("a typescript toolchain in the go build module's bucket counted")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
|
||||
t.Error("garbage or an unknown module counted")
|
||||
}
|
||||
if len(s.Entries("go")) != 0 {
|
||||
t.Error("an entry was kept")
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,14 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -23,15 +25,16 @@ import (
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
@@ -55,6 +58,84 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.BuildAgainTool,
|
||||
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
||||
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
||||
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
||||
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
||||
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
||||
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Revision == 0 && !r.Pending {
|
||||
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
||||
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
||||
}},
|
||||
{Name: delivery.BuildsSinceTool,
|
||||
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
|
||||
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
|
||||
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
|
||||
"log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
|
||||
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
|
||||
}
|
||||
return build.BuildsSince(since, at)
|
||||
}},
|
||||
{Name: build.AnnounceAgainTool,
|
||||
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
|
||||
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
|
||||
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
|
||||
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
|
||||
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused || r.Revision == 0 {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
@@ -85,8 +166,17 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
||||
a, err := build.DecodeQueueAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
}
|
||||
|
||||
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
||||
// bound for a value written after it that answered says is the builder's answer.
|
||||
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
answered := make(chan json.RawMessage, 1)
|
||||
heard := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
@@ -96,36 +186,34 @@ func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||
if answered(c.Value) {
|
||||
select {
|
||||
case answered <- c.Value:
|
||||
case heard <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||
return e.Value, nil
|
||||
}
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
||||
return e.Value, nil
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-answered:
|
||||
case v := <-heard:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -166,12 +254,17 @@ func main() {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
||||
a, err := build.DecodeReAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -123,5 +123,6 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
|
||||
return catalogue.Decode([]byte(kept.JSON))
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||
// controller had never taken in, and the builder kept them done.
|
||||
|
||||
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
|
||||
// answer}}; a reference is gone while gone says so.
|
||||
type heldVerb struct {
|
||||
mu sync.Mutex
|
||||
@@ -29,7 +30,7 @@ type heldVerb struct {
|
||||
|
||||
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||
t.Helper()
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
References []string `json:"references"`
|
||||
}
|
||||
|
||||
@@ -22,8 +22,13 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
// The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1),
|
||||
// another repository's, by its published version, so the builder is tested against the real pulling code.
|
||||
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
@@ -51,7 +56,7 @@ const (
|
||||
)
|
||||
|
||||
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
|
||||
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// its build seat's asks on its nodes (runner.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
|
||||
type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
@@ -66,14 +71,14 @@ type fakeBuildModules struct {
|
||||
// closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks
|
||||
// waiting in their streams).
|
||||
closed atomic.Bool
|
||||
pullers []*buildstep.Puller
|
||||
pullers []*runner.Puller
|
||||
}
|
||||
|
||||
// open lets the nodes pull again and wakes their pullers.
|
||||
func (f *fakeBuildModules) open() {
|
||||
f.closed.Store(false)
|
||||
f.mu.Lock()
|
||||
ps := append([]*buildstep.Puller(nil), f.pullers...)
|
||||
ps := append([]*runner.Puller(nil), f.pullers...)
|
||||
f.mu.Unlock()
|
||||
for _, p := range ps {
|
||||
p.Wake()
|
||||
@@ -137,26 +142,26 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
language := strings.TrimPrefix(module, "mesh-build-")
|
||||
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
return kv.Put(f.ctx, key, raw)
|
||||
}
|
||||
call := func(name string) buildstep.StepFunc {
|
||||
call := func(name string) runner.StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
|
||||
}
|
||||
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
steps := runner.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
for _, node := range nodes {
|
||||
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
p := &runner.Puller{Source: &runner.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
Capacity: func() int {
|
||||
if f.closed.Load() {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }}
|
||||
f.mu.Lock()
|
||||
f.pullers = append(f.pullers, p)
|
||||
f.mu.Unlock()
|
||||
@@ -267,6 +272,12 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
|
||||
logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"},
|
||||
AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||
b.fake.closed.Store(closed)
|
||||
b.fake.serve(t, "mesh-build-go", "n1", "n2")
|
||||
@@ -279,6 +290,29 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
return b
|
||||
}
|
||||
|
||||
// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user.
|
||||
func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry {
|
||||
t.Helper()
|
||||
conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []delivery.BuildLogEntry
|
||||
for _, le := range entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(le.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
||||
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
||||
t.Helper()
|
||||
@@ -345,7 +379,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -405,6 +439,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
|
||||
t.Errorf("toolchains = %+v", v.Toolchains)
|
||||
}
|
||||
// 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a
|
||||
// reader with the log's read reads them, the end naming the manifest and what was made.
|
||||
ended := b.logged(t, "netcheck")
|
||||
if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent ||
|
||||
ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" {
|
||||
t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended)
|
||||
}
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
|
||||
if got := b.fake.count(tool); got != want {
|
||||
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
|
||||
@@ -429,15 +470,20 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4).
|
||||
func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "."
|
||||
// is given to the build tools as it is (1615/AC4). One table: the two tests were merged to pay for the enrolment's.
|
||||
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
for _, c := range []struct{ name, path string }{{"no toolchain", "modules/ca-trust"}, {"root", delivery.RootPath}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", c.path, commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -492,14 +538,6 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
|
||||
}
|
||||
}
|
||||
|
||||
// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
|
||||
func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", delivery.RootPath, commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
}
|
||||
|
||||
func mustKey(t *testing.T, module string) string {
|
||||
k, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,12 +6,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -24,7 +18,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -51,80 +45,18 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
cred, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
conn, js, err := cred.Dial()
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -149,20 +81,37 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
var buildModules []string
|
||||
for _, l := range delivery.AskLanguages {
|
||||
if m, err := delivery.BuildModuleOf(l); err == nil {
|
||||
buildModules = append(buildModules, m)
|
||||
}
|
||||
}
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
@@ -171,6 +120,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
@@ -180,6 +130,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since.
|
||||
b.Log = func(e delivery.BuildLogEntry) {
|
||||
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
@@ -204,11 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
watch.put = gate.Toolchain
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
|
||||
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -220,28 +177,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// module.
|
||||
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||
// The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the
|
||||
// builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says
|
||||
// so; it still asks held at its start and at every module value.
|
||||
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
|
||||
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
|
||||
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
|
||||
default:
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -280,24 +235,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
@@ -306,7 +262,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
|
||||
@@ -8,6 +8,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -17,8 +20,14 @@ import (
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
|
||||
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
|
||||
}
|
||||
|
||||
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
|
||||
// longer for the builder's answer.
|
||||
const buildAgainBound = 15 * time.Second
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
@@ -34,6 +43,15 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeBuild {
|
||||
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
|
||||
go builtAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeAnnounce {
|
||||
announcedAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
@@ -53,12 +71,12 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
|
||||
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
@@ -73,3 +91,75 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
|
||||
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
|
||||
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
case res.Failed != "":
|
||||
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
|
||||
default:
|
||||
r.Pending = res.Revision == 0
|
||||
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
|
||||
if res.Pending != "" {
|
||||
r.Answer = res.Pending
|
||||
if res.Revision != 0 {
|
||||
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
|
||||
// unchanged, and the value is written back with the new revision, or why nothing was done.
|
||||
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
|
||||
if res.Refused != "" {
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
} else {
|
||||
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key
|
||||
// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential.
|
||||
// The id makes a repeated append of one entry one entry.
|
||||
func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
|
||||
data, err := delivery.EncodeBuildLogEntry(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := delivery.BuildLogKey(e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key)
|
||||
msg.Data = data
|
||||
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
|
||||
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
|
||||
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
|
||||
// issue 774).
|
||||
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,26 +9,31 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
|
||||
// eventsStream is the stream every module's and role's events are on, where the grants part makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-sdk buscompose EventsStream).
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
|
||||
const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held"
|
||||
|
||||
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
|
||||
type toolCaller func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
|
||||
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
|
||||
// the key is then left done, and the controller's take-in still refuses a gone blob.
|
||||
// storedVia is build.Builder's Stored: it asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2, issue 1937)
|
||||
// and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: the key
|
||||
// is then left done, and asked again at the next module value or artifacts-gone.
|
||||
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
|
||||
return func(_ context.Context, references []string) ([]string, error) {
|
||||
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
|
||||
body := map[string]any{"references": references}
|
||||
raw, err := call(SeatHeld, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
|
||||
return nil, fmt.Errorf("asking held: %w", err)
|
||||
}
|
||||
a, err := delivery.DecodeHeldAnswer(raw, references)
|
||||
if err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -3,12 +3,15 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
@@ -20,3 +23,5 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
|
||||
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
|
||||
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
|
||||
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
|
||||
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
|
||||
|
||||
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
|
||||
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
|
||||
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch cur.State {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
|
||||
default:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
|
||||
}
|
||||
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
|
||||
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
|
||||
}
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
|
||||
}
|
||||
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
|
||||
return BuildAgainResult{Revision: nrev}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
key *delivery.BuilderBuild
|
||||
fp string
|
||||
refuse string
|
||||
}{
|
||||
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
|
||||
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
|
||||
{name: "a key with no build is refused", refuse: "has no build key"},
|
||||
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
before := bytes.Clone(r.st.vals[r.key])
|
||||
fp := fp1
|
||||
if c.fp != "" {
|
||||
fp = c.fp
|
||||
}
|
||||
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,10 +6,12 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -20,7 +22,8 @@ import (
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
@@ -175,6 +178,42 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
|
||||
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
|
||||
// ask again, so a lone node off the agreed toolchain cannot spin on it.
|
||||
const (
|
||||
DefaultDeclineBound = 30 * time.Minute
|
||||
DefaultDeclinePause = 5 * time.Second
|
||||
)
|
||||
|
||||
func (b *Builder) declineBound() time.Duration {
|
||||
if b.DeclineBound > 0 {
|
||||
return b.DeclineBound
|
||||
}
|
||||
return DefaultDeclineBound
|
||||
}
|
||||
|
||||
func (b *Builder) declinePause() time.Duration {
|
||||
if b.DeclinePause != 0 {
|
||||
return max(b.DeclinePause, 0)
|
||||
}
|
||||
return DefaultDeclinePause
|
||||
}
|
||||
|
||||
// declinedWords names the nodes that declined an ask and why, in the order of their names.
|
||||
func declinedWords(by map[string]string) string {
|
||||
nodes := make([]string, 0, len(by))
|
||||
for n := range by {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var parts []string
|
||||
for _, n := range nodes {
|
||||
parts = append(parts, n+": "+by[n])
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
@@ -253,6 +292,8 @@ func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
|
||||
errDeclined = errors.New("declined")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
@@ -266,11 +307,42 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
var firstDecline time.Time
|
||||
declines := map[string]string{} // node -> why it declined, the last
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errDeclined):
|
||||
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
|
||||
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
|
||||
// checker's decline loop is, issue 890).
|
||||
now := time.Now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
declines[step.Node] = why
|
||||
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
|
||||
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
|
||||
}
|
||||
// The pause is a context's bound, not a timer (ADR 0437 §8).
|
||||
pause, stop := context.WithTimeout(ctx, b.declinePause())
|
||||
<-pause.Done()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return step, ask, ctx.Err()
|
||||
}
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
@@ -315,6 +387,16 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
|
||||
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
|
||||
// after a pause, within DeclineBound.
|
||||
return step, step.Reason, errDeclined
|
||||
}
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The build runner a build node runs an ask with is mesh-build-go's (novox/hq ADR 0593 §1), another repository's,
|
||||
// which the builder does not import. These tests run an ask as the builder needs it run: fetch once at the job's
|
||||
// fetch arguments with its package versions, build or copy each artifact, answer each one's digest and reference. They
|
||||
// test the builder, not the runner.
|
||||
|
||||
// stepFunc is one step of a build module, as its tool answers it.
|
||||
type stepFunc func(ctx context.Context, args map[string]any) (any, error)
|
||||
|
||||
// askSteps are a build module's steps an ask is run with.
|
||||
type askSteps struct{ Fetch, Build, Copy stepFunc }
|
||||
|
||||
// runAsk runs a build ask with steps and answers its delivery.BuildAnswer, or the error that fails the step.
|
||||
func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build == nil {
|
||||
return nil, fmt.Errorf("the ask %s holds no build job", ask.ID)
|
||||
}
|
||||
job := *ask.Build
|
||||
fetch := job.Fetch
|
||||
if len(ask.Packages) > 0 && fetch != nil {
|
||||
fetch = maps.Clone(fetch)
|
||||
fetch["packages"] = maps.Clone(ask.Packages)
|
||||
}
|
||||
answer := delivery.BuildAnswer{}
|
||||
workspace := ""
|
||||
for _, a := range job.Artifacts {
|
||||
var out any
|
||||
var err error
|
||||
if a.From != "" {
|
||||
if steps.Copy == nil {
|
||||
return nil, fmt.Errorf("this build module copies no artifact (%s)", a.Artifact)
|
||||
}
|
||||
out, err = steps.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
|
||||
} else {
|
||||
if workspace == "" {
|
||||
raw, err := asRaw(steps.Fetch(ctx, fetch))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Merge string `json:"merge"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
||||
return nil, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
||||
}
|
||||
if _, candidate := fetch["base"]; candidate && (len(f.Conflicts) > 0 || f.Merge == "") {
|
||||
return nil, fmt.Errorf("the head does not merge into the trunk: %s", strings.Join(f.Conflicts, ", "))
|
||||
}
|
||||
workspace = f.Workspace
|
||||
}
|
||||
args := map[string]any{"workspace": workspace, "module": job.Path, "artifact": a.Artifact}
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
out, err = steps.Build(ctx, args)
|
||||
}
|
||||
raw, err := asRaw(out, err)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, raw)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func asRaw(v any, err error) (json.RawMessage, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
|
||||
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
|
||||
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
|
||||
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
|
||||
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
|
||||
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
|
||||
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
|
||||
|
||||
// BuildAgainResult is the builder's answer to a request to build a key again.
|
||||
type BuildAgainResult struct {
|
||||
// Refused is why nothing was done, "" when the key was written building as a new revision.
|
||||
Refused string
|
||||
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
|
||||
Revision uint64
|
||||
Asks map[string]string
|
||||
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
|
||||
Pending string
|
||||
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
|
||||
// toolchain not agreed): the new revision was written, and nothing was asked.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// handWatch follows one request to build a key again until its asks are put.
|
||||
type handWatch struct {
|
||||
rev uint64
|
||||
lanes int
|
||||
asks map[string]string
|
||||
done chan struct{}
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
func (w *handWatch) settle() {
|
||||
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
|
||||
w.shut = true
|
||||
close(w.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) hand(key string, f func(w *handWatch)) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if w := b.hands[key]; w != nil {
|
||||
f(w)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) handClaimed(key string, rev uint64) {
|
||||
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handLanes(key string, n int) {
|
||||
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handAsk(key, language, id string) {
|
||||
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
|
||||
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
|
||||
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
|
||||
// ctx is the builder's own context: the build it starts lives on it.
|
||||
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
|
||||
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
|
||||
case cur.State != delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
|
||||
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
|
||||
}
|
||||
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
|
||||
// and the watch and the look are registered before anything else can take the key.
|
||||
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
|
||||
b.mu.Lock()
|
||||
m, known := b.known[module]
|
||||
switch {
|
||||
case !known || m.FP2 != fp:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
|
||||
case m.Manifest == "":
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
|
||||
case b.inflight[key] || b.hands[key] != nil:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
|
||||
}
|
||||
b.hands[key] = w
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
if b.hands[key] == w {
|
||||
delete(b.hands, key)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
|
||||
b.startOnce(ctx, key, module, m)
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
|
||||
defer stop()
|
||||
select {
|
||||
case <-w.done:
|
||||
case <-wait.Done():
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
// The look ended before every ask was put: say how the key stands, not that it was asked.
|
||||
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
|
||||
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
|
||||
switch cur.State {
|
||||
case delivery.BuildFailed:
|
||||
res.Failed = cur.Error
|
||||
case delivery.BuildDone:
|
||||
default:
|
||||
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
|
||||
}
|
||||
}
|
||||
}
|
||||
case !shut:
|
||||
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
|
||||
@@ -68,7 +68,7 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
continue
|
||||
}
|
||||
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest)}
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
|
||||
if len(cm.Paths) > 0 {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
|
||||
@@ -61,8 +61,8 @@ type ClaimOptions struct {
|
||||
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
|
||||
// clause 6). Nil means none is.
|
||||
Published func(ref string) bool
|
||||
// Stored answers which of the references given the artifact store no longer holds (the controller's verb held,
|
||||
// mesh-sdk delivery.HeldVerb; novox/hq issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// Stored answers which of the references given the artifact store no longer holds (the seat mesh-artifact-sweep's verb held,
|
||||
// novox/hq ADR 0571 §2, issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
|
||||
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
|
||||
@@ -29,7 +29,7 @@ func captured(t *testing.T, name string) json.RawMessage {
|
||||
return step.Result
|
||||
}
|
||||
|
||||
// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's
|
||||
// The captured image steps, run as a pulled ask runs them (runAsk), answer what the builder reads: the trunk's
|
||||
// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a).
|
||||
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
|
||||
for _, tc := range []struct {
|
||||
@@ -37,7 +37,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
publish bool
|
||||
ref bool
|
||||
}{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} {
|
||||
steps := buildstep.AskSteps{
|
||||
steps := askSteps{
|
||||
Fetch: func(context.Context, map[string]any) (any, error) {
|
||||
return json.RawMessage(captured(t, "fetch.json")), nil
|
||||
},
|
||||
@@ -47,7 +47,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish,
|
||||
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, ask)
|
||||
out, err := runAsk(context.Background(), steps, ask)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.build, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry,
|
||||
// where the indexer keeps them on the module.
|
||||
func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"}
|
||||
r.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
step := r.answer(a)
|
||||
var answer map[string]any
|
||||
if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" {
|
||||
answer["imports"] = imports
|
||||
step.Result, _ = json.Marshal(answer)
|
||||
}
|
||||
return step
|
||||
}
|
||||
var logged []delivery.BuildLogEntry
|
||||
var mu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() }
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") {
|
||||
t.Fatalf("done key %+v, want the imports", v)
|
||||
}
|
||||
if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 {
|
||||
t.Fatalf("log %+v, want the imports on the end alone", logged)
|
||||
}
|
||||
}
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
|
||||
|
||||
@@ -3,8 +3,8 @@ package build
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
||||
@@ -305,10 +305,9 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test
|
||||
r := newRig(t, tsPackage)
|
||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
||||
paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
|
||||
tree, err := delivery.NamedTree(manifestD, paths)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it,
|
||||
// never computes it.
|
||||
const tree = delivery.NamedTreePrefix + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281"
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".",
|
||||
Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}})
|
||||
if err != nil {
|
||||
|
||||
@@ -2,7 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
@@ -6,15 +6,17 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
||||
@@ -50,7 +52,7 @@ type Builder struct {
|
||||
// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
|
||||
// rests on it. Nil tells nobody.
|
||||
Answered func(buildModule string)
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
@@ -66,8 +68,16 @@ type Builder struct {
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
|
||||
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
|
||||
DeclineBound, DeclinePause time.Duration
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
|
||||
Log func(delivery.BuildLogEntry)
|
||||
// Now is the clock of the log; nil is time.Now. Node is the node the builder runs on, which asked the build.
|
||||
Now func() time.Time
|
||||
Node string
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -89,6 +99,8 @@ type Builder struct {
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -106,6 +118,7 @@ func (b *Builder) init() {
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -248,8 +261,8 @@ func (b *Builder) storedOnce(key string) func(ctx context.Context, references []
|
||||
}
|
||||
}
|
||||
|
||||
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
|
||||
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
|
||||
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
|
||||
// builder: what is built rests on the store's answer, never on the event alone.
|
||||
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
|
||||
@@ -372,20 +385,45 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
|
||||
}()
|
||||
}
|
||||
|
||||
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
|
||||
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
|
||||
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
defer b.handEnded(key)
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
if planErr == nil && !nothing {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
@@ -400,6 +438,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now()))
|
||||
if cl.Unasked != nil {
|
||||
b.logf("%v", cl.Unasked)
|
||||
}
|
||||
@@ -449,6 +489,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
b.logf("%s: the build could not be written: %v", cl.Key, err)
|
||||
return
|
||||
}
|
||||
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
}
|
||||
@@ -469,10 +510,11 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
@@ -492,6 +534,7 @@ type lane struct {
|
||||
toolchain *delivery.ToolchainUsed
|
||||
digests map[string]string // artifact -> digest
|
||||
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
|
||||
imports []string // the mesh packages the lane's programs import, as its build module answered them
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -506,7 +549,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
for _, bm := range plan.Modules() {
|
||||
if prior != nil && len(only) > 0 && !only[bm] {
|
||||
kept = append(kept, bm)
|
||||
@@ -516,10 +560,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
sitting = append(sitting, SittingOutWords(language, ag)...)
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
@@ -535,6 +581,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||
}
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.handLanes(key, len(lanes))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
@@ -613,10 +662,14 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
}
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(sitting) > 0 {
|
||||
step += " (" + strings.Join(sitting, "; ") + ")"
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
|
||||
Imports: importsOf(lanes)}
|
||||
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
|
||||
// A reference the build module answered that does not name the artifact and its digest is no reference.
|
||||
return failed("%s%v", delivery.NotStored, err)
|
||||
@@ -717,7 +770,11 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
v.Asks = map[string]string{}
|
||||
}
|
||||
v.Asks[language] = id
|
||||
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
|
||||
return err
|
||||
}
|
||||
n.b.handAsk(n.cl.Key, language, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||
@@ -764,6 +821,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
job.Fetch = req.Args()
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
|
||||
if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
}
|
||||
if id := asked.resume(language); id != "" {
|
||||
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||
if b.Held != nil {
|
||||
@@ -807,6 +869,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
|
||||
return
|
||||
}
|
||||
l.imports = answer.Imports
|
||||
built := map[string]delivery.AskBuilt{}
|
||||
for _, a := range answer.Artifacts {
|
||||
built[a.Artifact] = a
|
||||
@@ -836,3 +899,87 @@ func needsFetch(tasks []Task) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *Builder) now() time.Time {
|
||||
if b.Now != nil {
|
||||
return b.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// logBuild tells the log an entry, said in the journal when it cannot be one.
|
||||
func (b *Builder) logBuild(e delivery.BuildLogEntry) {
|
||||
if b.Log == nil {
|
||||
return
|
||||
}
|
||||
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
|
||||
b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err)
|
||||
return
|
||||
}
|
||||
b.Log(e)
|
||||
}
|
||||
|
||||
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
|
||||
func (b *Builder) refOf(module, fp string) string {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.via[key].Ref
|
||||
}
|
||||
|
||||
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
|
||||
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
|
||||
// that asked it and the time.
|
||||
func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry {
|
||||
dir := m.Path
|
||||
if dir == "" {
|
||||
dir = delivery.RootPath
|
||||
}
|
||||
return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10),
|
||||
Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()}
|
||||
}
|
||||
|
||||
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
|
||||
// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from.
|
||||
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
|
||||
e := LogStarted(module, m, ref, by, at)
|
||||
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
|
||||
if result.Manifest != "" {
|
||||
e.Manifest = result.Manifest
|
||||
}
|
||||
if m.Inputs2 != nil {
|
||||
e.Against = append([]string(nil), m.Inputs2.Bases...)
|
||||
}
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
|
||||
return e
|
||||
}
|
||||
|
||||
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
|
||||
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
|
||||
// kept one reports only what it built.
|
||||
func importsOf(lanes []*lane) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lanes {
|
||||
for _, p := range l.imports {
|
||||
seen[p] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
|
||||
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
|
||||
func SittingOutWords(language string, a delivery.Agreement) []string {
|
||||
var out []string
|
||||
for _, s := range a.SittingOut {
|
||||
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -13,7 +13,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -138,7 +139,7 @@ type rig struct {
|
||||
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
|
||||
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
|
||||
|
||||
// answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the
|
||||
// answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the
|
||||
// rig's pulling node, and answers the step that node writes.
|
||||
func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
node := r.puller
|
||||
@@ -147,7 +148,7 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
}
|
||||
module, _ := delivery.BuildModuleOf(a.Language)
|
||||
runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}
|
||||
tool := func(name string) buildstep.StepFunc {
|
||||
tool := func(name string) stepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||
raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args)
|
||||
if err != nil {
|
||||
@@ -156,8 +157,8 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
return raw, nil
|
||||
}
|
||||
}
|
||||
steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
steps := askSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
|
||||
out, err := runAsk(context.Background(), steps, a)
|
||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
@@ -220,8 +221,52 @@ var (
|
||||
)
|
||||
|
||||
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
|
||||
t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog)
|
||||
r := newRig(t, goBundle, archiveA, tsBundle)
|
||||
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
|
||||
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
|
||||
defer func() {
|
||||
if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
|
||||
t.Errorf("the key does not say ace sits out: %s", st)
|
||||
}
|
||||
}()
|
||||
var logged []delivery.BuildLogEntry
|
||||
var lmu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
|
||||
v := r.consider(t, "modules/m")
|
||||
// 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read
|
||||
// from the two.
|
||||
if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent ||
|
||||
logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" ||
|
||||
logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" ||
|
||||
logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" ||
|
||||
logged[0].FP != fp1 || logged[0].Module != "m" {
|
||||
t.Fatalf("logged = %+v", logged)
|
||||
}
|
||||
for _, e := range logged {
|
||||
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
|
||||
t.Errorf("an entry the log refuses: %v", err)
|
||||
}
|
||||
}
|
||||
if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok {
|
||||
t.Error("no duration from the two entries")
|
||||
}
|
||||
// builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more.
|
||||
var page stdio.LogPage
|
||||
for i, e := range logged {
|
||||
raw, _ := delivery.EncodeBuildLogEntry(e)
|
||||
page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At})
|
||||
}
|
||||
page.Next = 3
|
||||
since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At)
|
||||
if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More {
|
||||
t.Errorf("builds since = %+v, %v, want the one end and more", since, err)
|
||||
}
|
||||
page.Entries[0].Value = []byte(`{"v":0}`)
|
||||
if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil {
|
||||
t.Error("an entry that does not read was skipped, want it refused")
|
||||
}
|
||||
if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
|
||||
t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
|
||||
}
|
||||
@@ -264,7 +309,14 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
||||
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil // no node has pulled it yet
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
|
||||
sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
var asks []delivery.StepAsk
|
||||
for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
@@ -276,6 +328,9 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
|
||||
t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
|
||||
}
|
||||
if !reflect.DeepEqual(a.Packages, sdk) {
|
||||
t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
|
||||
t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
|
||||
}
|
||||
@@ -683,6 +738,25 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
|
||||
waitForAsks(t, r, 1)
|
||||
for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
|
||||
r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
|
||||
ended++
|
||||
}
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
|
||||
t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
@@ -690,17 +764,130 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
deaf bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
|
||||
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
|
||||
// is refused, "" when it is built again as a new revision and its ask id answered.
|
||||
again bool
|
||||
key *delivery.BuilderBuild
|
||||
known string
|
||||
refuse string
|
||||
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
|
||||
// the error the key then holds when its build ends before its asks are put.
|
||||
edit func(m *delivery.Module)
|
||||
busy int
|
||||
failed string
|
||||
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
|
||||
during bool
|
||||
}{
|
||||
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
|
||||
edit: func(m *delivery.Module) {
|
||||
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
|
||||
},
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
|
||||
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
|
||||
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
|
||||
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
|
||||
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
|
||||
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
if c.again {
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.known != "" {
|
||||
m.FP2 = c.known
|
||||
}
|
||||
if c.edit != nil {
|
||||
c.edit(&m)
|
||||
}
|
||||
r.b.init()
|
||||
var stored atomic.Int64
|
||||
if c.during {
|
||||
r.asks.Answer = nil
|
||||
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
|
||||
}
|
||||
switch c.busy {
|
||||
case 1:
|
||||
r.b.inflight[r.key] = true
|
||||
case 2:
|
||||
r.b.hands[r.key] = &handWatch{}
|
||||
}
|
||||
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
|
||||
log := &logged{}
|
||||
r.b.Logf = log.logf
|
||||
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
|
||||
if c.during {
|
||||
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
|
||||
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
|
||||
}
|
||||
return
|
||||
}
|
||||
r.b.Wait()
|
||||
after := r.st.revs[r.key]
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
|
||||
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.failed != "" {
|
||||
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
|
||||
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
asks := r.asks.Asks()
|
||||
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
|
||||
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
|
||||
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
|
||||
}
|
||||
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
|
||||
t.Errorf("the why is not logged: %v", log.lines)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
@@ -714,7 +901,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
if !c.byHand {
|
||||
dead := &buildrun.DeadLetters{}
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
var heard *buildrun.HeardEnds
|
||||
if c.deaf {
|
||||
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead && !c.deaf {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
@@ -724,6 +920,10 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.dead {
|
||||
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||
}
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
@@ -732,7 +932,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.done {
|
||||
if c.deaf {
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask is waited on")
|
||||
}
|
||||
}
|
||||
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
|
||||
}
|
||||
if c.done && !c.deaf {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
@@ -757,13 +966,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
@@ -836,12 +1048,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
@@ -959,3 +1176,10 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
|
||||
}
|
||||
}
|
||||
|
||||
// silentWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type silentWatch struct{}
|
||||
|
||||
func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more.
|
||||
const BuildsSinceLimit = 500
|
||||
|
||||
// LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since.
|
||||
type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error)
|
||||
|
||||
// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on,
|
||||
// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again
|
||||
// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that
|
||||
// skipped it would register too little.
|
||||
func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) {
|
||||
out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}}
|
||||
page, err := read(since, "", BuildsSinceLimit)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err)
|
||||
}
|
||||
if page == nil {
|
||||
return out, nil
|
||||
}
|
||||
var entries []delivery.BuildLogEntry
|
||||
for _, le := range page.Entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(le.Value)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
if ended := delivery.EndedSince(entries, since); ended != nil {
|
||||
out.Builds = ended
|
||||
}
|
||||
out.More = page.Next != 0
|
||||
return out, nil
|
||||
}
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||
@@ -22,6 +22,11 @@ import (
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
|
||||
BuildAgainTool = "builder_build_again"
|
||||
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
|
||||
// of Mode ModeAnnounce.
|
||||
AnnounceAgainTool = "builder_announce_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
@@ -36,8 +41,27 @@ type ReAsk struct {
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
|
||||
// revision (building) and the ask ids it put by language.
|
||||
Refused bool `json:"refused,omitempty"`
|
||||
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
|
||||
// Failed is the error of a key written building that failed before its asks were put.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Revision uint64 `json:"revision,omitempty"`
|
||||
Asks map[string]string `json:"asks,omitempty"`
|
||||
}
|
||||
|
||||
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
|
||||
const ModeBuild = "build"
|
||||
|
||||
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
|
||||
const ModeAnnounce = "announce"
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
@@ -63,6 +87,32 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error)
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
|
||||
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the key is built again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
|
||||
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the done build is announced again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
@@ -106,7 +156,9 @@ type StuckReader struct {
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
Now func() time.Time
|
||||
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||
Hearing StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
@@ -149,6 +201,20 @@ func parallel(n int, read func(i int) error) error {
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||
func (r StuckReader) deadLetters() map[string]any {
|
||||
if r.Hearing.Get == nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||
}
|
||||
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||
}
|
||||
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
@@ -268,6 +334,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
answer["dead_letters"] = r.deadLetters()
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
@@ -40,10 +42,14 @@ type manifest struct {
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Logs []struct {
|
||||
Name string `json:"name"`
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
} `json:"logs"`
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Pattern string `json:"pattern"`
|
||||
@@ -87,13 +93,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
"mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
@@ -101,12 +107,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
}
|
||||
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
|
||||
// other tool; it holds no capacity.
|
||||
if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 {
|
||||
// 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955).
|
||||
if want := []string{"seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
|
||||
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
|
||||
}
|
||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
|
||||
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
|
||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
// starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the
|
||||
// controller's no longer (1126/B9, ADR 0590).
|
||||
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -116,13 +124,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
// 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream.
|
||||
if len(m.Logs) != 1 || m.Logs[0].Name != delivery.BuilderLog || m.Logs[0].MaxAgeDays != delivery.BuilderLogDays {
|
||||
t.Errorf("logs = %+v, want %s kept %d days", m.Logs, delivery.BuilderLog, delivery.BuilderLogDays)
|
||||
}
|
||||
for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_announce_again",
|
||||
"builder_ask_again",
|
||||
"builder_build_again",
|
||||
"builder_builds_since",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
@@ -11,26 +14,30 @@
|
||||
"build.asked-again"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-artifact-sweep.artifacts-gone",
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
"mesh-controller.artifacts-gone"
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.held"
|
||||
"seat:mesh-artifact-sweep.held"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
],
|
||||
"logs": [
|
||||
{ "name": "build-log", "max-age-days": 30 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
@@ -41,7 +48,6 @@
|
||||
"node-build-image",
|
||||
"node-build-node"
|
||||
],
|
||||
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
|
||||
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
|
||||
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
// state and reads. It writes only the tries (the try's tools) and, asking a check again by hand, one verdict. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -53,10 +54,36 @@ func read(s stdio.KeptState) check.StateRead {
|
||||
}}
|
||||
}
|
||||
|
||||
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||
type runsOf struct{ modules check.StateRead }
|
||||
|
||||
func (r runsOf) CommitOf(module string) string {
|
||||
if r.modules.Get == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := r.modules.Get(module)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// stateStore is a state the tools write: the tries.
|
||||
type stateStore struct {
|
||||
check.StateRead
|
||||
create func(key string, value []byte) error
|
||||
entry func(key string) (json.RawMessage, uint64, error)
|
||||
update func(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||
return s.update(key, value, revision)
|
||||
}
|
||||
|
||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||
@@ -66,19 +93,21 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||
type Trying struct {
|
||||
Tries check.TryStore
|
||||
Ender check.TryEnder
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
||||
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||
func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -106,7 +135,8 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -115,14 +145,51 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if tr.Ender == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||
}
|
||||
now := time.Now
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try,
|
||||
// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them.
|
||||
func SeatTryTools(tr Trying) []stdio.Tool {
|
||||
verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"}
|
||||
var out []stdio.Tool
|
||||
for _, t := range TryTools(tr) {
|
||||
t.Name = Seat + "." + verbs[t.Name]
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Tools are the read tools over the checker's states.
|
||||
func Tools(r check.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: check.VerdictsReadTool,
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, every failure it names one line each (every member's failed tests, refused steps and conflicts, the manifests' refusals and the composition), the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
@@ -132,7 +199,7 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
Description: "The verdict that judges one pull request, every failure it names one line each (all of them, not only the first the status line shows), and the full report of its status mesh/check. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
||||
@@ -174,12 +241,33 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
}
|
||||
|
||||
// RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose
|
||||
// wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w.
|
||||
func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{{Name: check.AskAgainTool,
|
||||
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
|
||||
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
|
||||
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
|
||||
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
|
||||
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return r.AskAgain(w, str(args, "group"), str(args, "why"))
|
||||
}}}
|
||||
}
|
||||
|
||||
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
|
||||
// duration.
|
||||
func stepBound() time.Duration {
|
||||
@@ -193,7 +281,10 @@ func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
@@ -205,14 +296,35 @@ func main() {
|
||||
return s.Heads, err
|
||||
}
|
||||
r.Trunks = trunks
|
||||
trying := Trying{
|
||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := tries.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
verdicts := stdio.State("verdicts")
|
||||
written := stateStore{entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := verdicts.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,14 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("tools = %v, want %v", names, want)
|
||||
}
|
||||
// The recovery tool beside them asks a lost check again; with nothing to write it refuses.
|
||||
recovery := RecoveryTools(check.Reader{}, nil)
|
||||
if len(recovery) != 1 || recovery[0].Name != check.AskAgainTool || recovery[0].Description == "" {
|
||||
t.Errorf("recovery tools = %+v", recovery)
|
||||
}
|
||||
if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil {
|
||||
t.Errorf("%s answered with no state to write", check.AskAgainTool)
|
||||
}
|
||||
// With nothing to read from a tool says so and changes nothing.
|
||||
for _, tl := range Tools(check.Reader{}) {
|
||||
if tl.Name == check.StatusReadTool {
|
||||
@@ -53,13 +61,19 @@ func (m *memStore) Create(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||
m.vals[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
||||
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
@@ -74,6 +88,32 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike.
|
||||
verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" {
|
||||
t.Fatalf("verbs = %+v", verbs)
|
||||
}
|
||||
again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
if err != nil || again.(map[string]any)["id"] != id {
|
||||
t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id)
|
||||
}
|
||||
if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) {
|
||||
t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw)
|
||||
}
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
t.Errorf("end = %v, %v", ended, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||
t.Error("an ended try was ended again")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||
t.Errorf("hq: %v", err)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -124,9 +125,61 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
return parseManifest(kept.JSON)
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
|
||||
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
|
||||
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
|
||||
// parseManifest decodes a manifest the indexer kept, as every reader does (novox/hq ADR 0578 §3): judged by nothing here,
|
||||
// a field it does not know set aside. The checker judges a change's manifests with the indexer's judge, run as a step
|
||||
// (judge.go).
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
return catalogue.Decode([]byte(raw))
|
||||
}
|
||||
|
||||
// awaitedKV is a bucket that may appear after the start (process.AwaitBucket) as a check.KV: a call while it is not on the
|
||||
// bus fails saying so.
|
||||
type awaitedKV struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (a awaitedKV) kv() (kvStore, error) {
|
||||
kv := a.b.KV()
|
||||
if kv == nil {
|
||||
return kvStore{}, errors.New("the bucket is not on the bus yet")
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
}
|
||||
|
||||
func (a awaitedKV) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return s.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Keys(ctx context.Context) ([]string, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Keys(ctx)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Delete(ctx context.Context, key string) error {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Delete(ctx, key)
|
||||
}
|
||||
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// assignerWrote raises the assigner's compositions bucket, the assigner online in the watchman's presence, and its verdict
|
||||
// under the check's key, as the assigner leaves them on the bus.
|
||||
func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerdict) {
|
||||
t.Helper()
|
||||
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, pkey, on); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeCompositionVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(b.ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
|
||||
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
|
||||
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
state, owed, words string
|
||||
}{
|
||||
{delivery.CompositionPass, "success", "composition passed"},
|
||||
{delivery.CompositionFail, "failure", "g14 runs web, which the change removes from its source"},
|
||||
} {
|
||||
b := startBed(t)
|
||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
|
||||
v := delivery.CompositionVerdict{Group: group, Heads: heads, State: c.state, SDK: "v0.1.14-x", Machines: []string{"g14"}}
|
||||
if c.state == delivery.CompositionFail {
|
||||
v.Machine, v.Words = "g14", c.words
|
||||
}
|
||||
b.assignerWrote(t, key, v)
|
||||
b.start(t, nil)
|
||||
got := b.verdict(t, key, ended)
|
||||
if got.Composition == nil || got.Composition.State != c.state {
|
||||
t.Fatalf("%s: the verdict's composition is %+v", c.state, got.Composition)
|
||||
}
|
||||
o := b.owed(t, 7, head1, c.owed)
|
||||
if !strings.Contains(o.Text, c.words) || strings.Contains(o.Text, "not yet run") {
|
||||
t.Errorf("%s: the line says %q", c.state, o.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// compositionWatch reads the assigner's composition of a check (ADR 0528 clause 5) by watching its key and the
|
||||
// assigner's presence: no poll and no timer. With no assigner online it answers check.ErrAssignerNotRunning at once.
|
||||
type compositionWatch struct {
|
||||
kv jetstream.KeyValue
|
||||
// presence is the watchman's presence bucket; nil (not there) counts as no assigner online.
|
||||
presence jetstream.KeyValue
|
||||
}
|
||||
|
||||
// Await answers the composition under key once the assigner wrote one that does not wait (check.AwaitComposition).
|
||||
func (c compositionWatch) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
comps := make(chan check.CompositionEvent)
|
||||
presence := make(chan check.PresenceEvent)
|
||||
cw, err := c.kv.Watch(ctx, key)
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{}, err
|
||||
}
|
||||
defer cw.Stop()
|
||||
go func() {
|
||||
for e := range cw.Updates() {
|
||||
ev := check.CompositionEvent{Settled: e == nil}
|
||||
if e != nil && e.Operation() == jetstream.KeyValuePut {
|
||||
ev.Value = e.Value()
|
||||
}
|
||||
select {
|
||||
case comps <- ev:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
if c.presence == nil {
|
||||
go func() {
|
||||
select {
|
||||
case presence <- check.PresenceEvent{Settled: true}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
pw, err := c.presence.Watch(ctx, "part."+delivery.AssignerModule+".*")
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{}, err
|
||||
}
|
||||
defer pw.Stop()
|
||||
go func() {
|
||||
for e := range pw.Updates() {
|
||||
ev := check.PresenceEvent{Settled: e == nil}
|
||||
if e != nil {
|
||||
ev.Key = e.Key()
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
if p, err := delivery.DecodePresence(e.Value()); err == nil {
|
||||
ev.Online = p.Online
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case presence <- ev:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
return check.AwaitComposition(ctx, presence, comps)
|
||||
}
|
||||
|
||||
// followCompositions hands the checker each composition the assigner writes from now on, so a verdict that said its
|
||||
// composition was not judged takes it in when the assigner starts (event-driven).
|
||||
func followCompositions(ctx context.Context, kv jetstream.KeyValue, ch *check.Checker) error {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for e := range w.Updates() {
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
if comp, err := delivery.DecodeCompositionVerdict(e.Value()); err == nil {
|
||||
ch.Composed(ctx, comp)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// lazyCompositions opens the assigner's compositions when a check first asks for one and the bucket is there: until it
|
||||
// is, no assigner ever ran, so the check says at once that its composition is not judged (check.ErrAssignerNotRunning),
|
||||
// and the next check looks again. No timer: each look is a check asking.
|
||||
type lazyCompositions struct {
|
||||
open func(ctx context.Context) (check.CompositionReader, error)
|
||||
|
||||
mu sync.Mutex
|
||||
reader check.CompositionReader
|
||||
}
|
||||
|
||||
// Await is the opened reader's Await, opening it first when it is not yet.
|
||||
func (l *lazyCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||
l.mu.Lock()
|
||||
r := l.reader
|
||||
if r == nil {
|
||||
opened, err := l.open(ctx)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
l.mu.Unlock()
|
||||
return delivery.CompositionVerdict{}, check.ErrAssignerNotRunning
|
||||
case err != nil:
|
||||
l.mu.Unlock()
|
||||
return delivery.CompositionVerdict{}, err
|
||||
}
|
||||
l.reader, r = opened, opened
|
||||
}
|
||||
l.mu.Unlock()
|
||||
return r.Await(ctx, key)
|
||||
}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
|
||||
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
|
||||
func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PresenceBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
if _, err := pkv.Put(ctx, pkey, on); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// With no assigner online the wait answers at once (a second key, before the assigner of this test is online).
|
||||
off, _ := delivery.EncodePresence(delivery.Presence{Online: false})
|
||||
if _, err := pkv.Put(ctx, pkey, off); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, _ := delivery.CompositionKey("feat/y", "0123456789ab")
|
||||
if _, err := (compositionWatch{kv: kv, presence: pkv}).Await(ctx, other); !errors.Is(err, check.ErrAssignerNotRunning) {
|
||||
t.Fatalf("with no assigner online the wait answered %v", err)
|
||||
}
|
||||
if _, err := pkv.Put(ctx, pkey, on); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.CompositionKey("feat/x", "0123456789ab")
|
||||
got := make(chan delivery.CompositionVerdict, 1)
|
||||
go func() {
|
||||
v, err := compositionWatch{kv: kv, presence: pkv}.Await(ctx, key)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
got <- v
|
||||
}()
|
||||
put := func(v delivery.CompositionVerdict) {
|
||||
data, err := delivery.EncodeCompositionVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "the indexer holds no candidates yet"})
|
||||
select {
|
||||
case v := <-got:
|
||||
t.Fatalf("a waiting composition was answered: %+v", v)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}})
|
||||
select {
|
||||
case v := <-got:
|
||||
if v.State != delivery.CompositionPass || v.Machines[0] != "g14" {
|
||||
t.Errorf("answered %+v", v)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the composition written was not answered")
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
type answering struct{ v delivery.CompositionVerdict }
|
||||
|
||||
func (a answering) Await(context.Context, string) (delivery.CompositionVerdict, error) {
|
||||
return a.v, nil
|
||||
}
|
||||
|
||||
// 1297 review item 3: the compositions bucket missing when the checker starts is looked up again at the next check: while
|
||||
// it is missing a check says at once the assigner is not running; once it is there the check reads its composition, and
|
||||
// it is opened once.
|
||||
func TestTheCompositionsBucketIsLookedUpAtEachCheck(t *testing.T) {
|
||||
opens, there := 0, false
|
||||
l := &lazyCompositions{open: func(context.Context) (check.CompositionReader, error) {
|
||||
opens++
|
||||
if !there {
|
||||
return nil, jetstream.ErrBucketNotFound
|
||||
}
|
||||
return answering{delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x"}}, nil
|
||||
}}
|
||||
if _, err := l.Await(context.Background(), "k"); !errors.Is(err, check.ErrAssignerNotRunning) {
|
||||
t.Fatalf("without the bucket: %v", err)
|
||||
}
|
||||
there = true
|
||||
for range 2 {
|
||||
v, err := l.Await(context.Background(), "k")
|
||||
if err != nil || v.State != delivery.CompositionPass {
|
||||
t.Fatalf("with the bucket: %+v, %v", v, err)
|
||||
}
|
||||
}
|
||||
if opens != 2 {
|
||||
t.Errorf("opened %d times, want twice (missing, then once)", opens)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The checker's composition of a check on the bus (novox/hq ADR 0593 §7, §9; check.EngineCompositions): each placed
|
||||
// node's engine asked `node-engine.compose-candidate` on that node, and, for a group that changes the node-engine's
|
||||
// composition code, the composer built from its head run on the Go build seat.
|
||||
|
||||
// composeCandidateKey is the seat verb's call key, asked of one node (process.Conn.CallOn).
|
||||
var composeCandidateKey = "seat:" + catalogue.EngineSeat + "." + delivery.EngineComposeCandidateVerb
|
||||
|
||||
// engineAsks asks one node's engine over the bus, its answer waited for at most ctx's deadline (the answer bound).
|
||||
type engineAsks struct {
|
||||
call func(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
func (e engineAsks) ComposeCandidate(ctx context.Context, node string, ask delivery.ComposeCandidateAsk) (delivery.ComposeCandidateAnswer, error) {
|
||||
type answer struct {
|
||||
raw json.RawMessage
|
||||
err error
|
||||
}
|
||||
got := make(chan answer, 1)
|
||||
go func() {
|
||||
raw, err := e.call(node, composeCandidateKey, ask)
|
||||
got <- answer{raw, err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.ComposeCandidateAnswer{}, fmt.Errorf("no answer within its bound: %w", ctx.Err())
|
||||
case a := <-got:
|
||||
if a.err != nil {
|
||||
return delivery.ComposeCandidateAnswer{}, a.err
|
||||
}
|
||||
return delivery.DecodeComposeCandidateAnswer(a.raw)
|
||||
}
|
||||
}
|
||||
|
||||
// placedNodes names every node the seat mesh-placement places now, read from its state when a check composes (no
|
||||
// watch: a check reads it once). A state that cannot be read is an error, so the check says its composition was not
|
||||
// made rather than passing with no node asked (review finding 4).
|
||||
type placedNodes struct {
|
||||
kv func(ctx context.Context) (jetstream.KeyValue, error)
|
||||
}
|
||||
|
||||
func (p placedNodes) Placed() ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), delivery.ComposeAnswerBound)
|
||||
defer cancel()
|
||||
kv, err := p.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", delivery.PlacementBucket, err)
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil // a placement that places nothing
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s cannot be listed: %w", delivery.PlacementBucket, err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for key := range lister.Keys() {
|
||||
if node, _, _, err := delivery.ParsePlacementKey(key); err == nil {
|
||||
seen[node] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for n := range seen {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seatSandbox runs a compose job on the Go build seat's check stream, as the checker's own ask, and answers the
|
||||
// composer's verdict from the step's result (delivery.ComposeAnswer).
|
||||
type seatSandbox struct{ asker buildrun.Asker }
|
||||
|
||||
func (s seatSandbox) Compose(ctx context.Context, job delivery.ComposeJob) (composer.Verdict, error) {
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
|
||||
At: time.Now().UTC(), Compose: &job}
|
||||
step, err := s.asker.Ask(ctx, ask)
|
||||
if err != nil {
|
||||
return composer.Verdict{}, err
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return composer.Verdict{}, fmt.Errorf("the compose ask %s ended %s: %s", ask.ID, step.State, step.Reason)
|
||||
}
|
||||
var a delivery.ComposeAnswer
|
||||
if err := json.Unmarshal(step.Result, &a); err != nil {
|
||||
return composer.Verdict{}, fmt.Errorf("the compose ask %s answered no compose answer: %w", ask.ID, err)
|
||||
}
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
|
||||
return composer.Verdict{}, fmt.Errorf("the compose ask %s answered no composer verdict of version %d (%v)", ask.ID, composer.Version, err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// manifestVerdicts are the indexer's refusals of the trunk's manifests, by module, from its modules' values
|
||||
// (delivery.Module.VerdictOf, ADR 0578 §3): what the sandbox's composer holds a refused module by.
|
||||
type manifestVerdicts struct {
|
||||
mu sync.Mutex
|
||||
by map[string]delivery.ManifestVerdict
|
||||
}
|
||||
|
||||
func (m *manifestVerdicts) put(module string, v *delivery.ManifestVerdict) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.by == nil {
|
||||
m.by = map[string]delivery.ManifestVerdict{}
|
||||
}
|
||||
if v == nil {
|
||||
delete(m.by, module)
|
||||
return
|
||||
}
|
||||
m.by[module] = *v
|
||||
}
|
||||
|
||||
func (m *manifestVerdicts) all() map[string]delivery.ManifestVerdict {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := make(map[string]delivery.ManifestVerdict, len(m.by))
|
||||
for k, v := range m.by {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -13,16 +13,9 @@ import (
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// The forge's tools the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text
|
||||
// of a file at a ref, and gitea_compare answers {total_commits, commits}, what head holds that base does not. The checker
|
||||
// writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2).
|
||||
const (
|
||||
getFileTool = "gitea.gitea_get_file"
|
||||
compareTool = "gitea.gitea_compare"
|
||||
commitsTool = "gitea.gitea_commits"
|
||||
// gitea_list_branches answers {branches: [{name, commit}]}, every branch with the commit it points at.
|
||||
branchesTool = "gitea.gitea_list_branches"
|
||||
)
|
||||
// The forge's tool the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text
|
||||
// of a file at a ref. The checker writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2).
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFoundStatus is the 404 of the forge's error, `Gitea API <path>: <status> <body>`.
|
||||
var notFoundStatus = regexp.MustCompile(`: 404( |$)`)
|
||||
@@ -69,61 +62,3 @@ func (f forgeReads) File(_ context.Context, repository, path, ref string) (strin
|
||||
}
|
||||
return *a.File.Content, nil
|
||||
}
|
||||
|
||||
// OnTrunk says whether the commit is already on the repository's main: main holds it when the commit holds nothing main does
|
||||
// not.
|
||||
func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, error) {
|
||||
owner, repo := ownerRepo(repository)
|
||||
raw, err := f.ask(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var a struct {
|
||||
Total *int `json:"total_commits"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.Total == nil {
|
||||
return false, fmt.Errorf("the answer of %s is not {total_commits}: %v", compareTool, err)
|
||||
}
|
||||
return *a.Total == 0, nil
|
||||
}
|
||||
|
||||
// FullSHA is the whole sha of a commit named by its first digits: the newest commit of the history at that ref.
|
||||
func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (string, error) {
|
||||
owner, repo := ownerRepo(repository)
|
||||
raw, err := f.ask(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var a struct {
|
||||
Commits []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"commits"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || len(a.Commits) == 0 || len(a.Commits[0].SHA) != 40 || !strings.HasPrefix(a.Commits[0].SHA, commit) {
|
||||
return "", fmt.Errorf("the answer of %s for %s is not a commit that starts with it: %v", commitsTool, commit, err)
|
||||
}
|
||||
return a.Commits[0].SHA, nil
|
||||
}
|
||||
|
||||
// BranchHeads is the commit each branch of the repository points at.
|
||||
func (f forgeReads) BranchHeads(_ context.Context, repository string) (map[string]string, error) {
|
||||
owner, repo := ownerRepo(repository)
|
||||
raw, err := f.ask(branchesTool, map[string]string{"owner": owner, "repo": repo})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Branches *[]struct {
|
||||
Name string `json:"name"`
|
||||
Commit string `json:"commit"`
|
||||
} `json:"branches"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.Branches == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {branches: [{name, commit}]}: %v", branchesTool, err)
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, b := range *a.Branches {
|
||||
heads[b.Name] = b.Commit
|
||||
}
|
||||
return heads, nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user