Compare commits
606
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
945906ac97 | ||
|
|
1c66d58bce | ||
|
|
34ef185ba5 | ||
|
|
f12823681c | ||
|
|
b96a4f1d99 | ||
|
|
70ac4ff79a | ||
|
|
51deed51e3 | ||
|
|
3842a464ed | ||
|
|
877b09e92b | ||
|
|
453cdfa607 | ||
|
|
8f7945a109 | ||
|
|
2773885600 | ||
|
|
45bd63427f | ||
|
|
effa12c4b4 | ||
|
|
9da14440e8 | ||
|
|
fd638b99d5 | ||
|
|
0d1edef846 | ||
|
|
5c4d770aa1 | ||
|
|
b4855f21c3 | ||
|
|
1387b10e78 | ||
|
|
a1814c6571 | ||
|
|
92b4307922 | ||
|
|
2c37f48f9d | ||
|
|
4a736858b6 | ||
|
|
3411b91571 | ||
|
|
0ce16ad773 | ||
|
|
875d42fe3e | ||
|
|
aacdba44e3 | ||
|
|
39b3ce3c73 | ||
|
|
e1d0c9fe03 | ||
|
|
6bc16d74db | ||
|
|
9f49bc92ad | ||
|
|
4f6e1021c6 | ||
|
|
027c445bae | ||
|
|
7ba2668aa8 | ||
|
|
a901b840a0 | ||
|
|
f70faed5b6 | ||
|
|
5ee3b4c15f | ||
|
|
2f387a3f4c | ||
|
|
22aedbe97f | ||
|
|
478f0110e9 | ||
|
|
6d24922172 | ||
|
|
0fa00d5e6c | ||
|
|
a41d2851c7 | ||
|
|
ea5be10532 | ||
|
|
6b9d211f72 | ||
|
|
b342c67b19 | ||
|
|
daf178c4ca | ||
|
|
a9aa6d29ea | ||
|
|
ecf96be8ce | ||
|
|
ab3cb1ca9e | ||
|
|
054f089614 | ||
|
|
6301e49c73 | ||
|
|
78fe1ce960 | ||
|
|
6645ac58c2 | ||
|
|
c4470f42db | ||
|
|
4bb790bedf | ||
|
|
aa98729457 | ||
|
|
2fdb0516cd | ||
|
|
28a6858863 | ||
|
|
1ab0274547 | ||
|
|
07af86b153 | ||
|
|
dd2e7c8e64 | ||
|
|
756a3813e2 | ||
|
|
9a508701ff | ||
|
|
7ee7248fad | ||
|
|
8f83608503 | ||
|
|
bedda86a22 | ||
|
|
0e26fd8ed5 | ||
|
|
a62276bcaf | ||
|
|
34a607a1f5 | ||
|
|
f73b0bf6f6 | ||
|
|
15d28ddf5b | ||
|
|
22f3658a36 | ||
|
|
073e3fab65 | ||
|
|
2b5a7fe831 | ||
|
|
3b3b096be8 | ||
|
|
6313f3ea10 | ||
|
|
712d136fe6 | ||
|
|
09ac178888 | ||
|
|
d4600efd7d | ||
|
|
0edd0bca74 | ||
|
|
2fb1dc3214 | ||
|
|
e5ce35cd00 | ||
|
|
e31da597bb | ||
|
|
33675f548a | ||
|
|
008a85b73c | ||
|
|
6fb743f521 | ||
|
|
c0209b3d8e | ||
|
|
c12804f4d9 | ||
|
|
fbb50f8fcd | ||
|
|
0ce3409141 | ||
|
|
46e53a81c2 | ||
|
|
7cf47c04eb | ||
|
|
8855735341 | ||
|
|
c62a501bf0 | ||
|
|
0b03b9c7c5 | ||
|
|
d283efdc9f | ||
|
|
74304052b4 | ||
|
|
2a91caddaf | ||
|
|
78fedabeb1 | ||
|
|
5c20b9ef15 | ||
|
|
fa85de8ef6 | ||
|
|
290e1696b2 | ||
|
|
b7551d2efb | ||
|
|
2e91f0b061 | ||
|
|
61b2b718e6 | ||
|
|
2e23b0b736 | ||
|
|
b81e05387b | ||
|
|
525d7374b2 | ||
|
|
4c4380b94b | ||
|
|
f6fa75e189 | ||
|
|
604f1528d6 | ||
|
|
2aa43ec249 | ||
|
|
fba98ee80f | ||
|
|
511e4b7414 | ||
|
|
4dd7dcfc14 | ||
|
|
d26781b403 | ||
|
|
e2cb398186 | ||
|
|
6a4d88265d | ||
|
|
655692c7ba | ||
|
|
8f9766e46c | ||
|
|
83c1b3ce38 | ||
|
|
000a24d261 | ||
|
|
3228d911cb | ||
|
|
7e358b0f1d | ||
|
|
8d4c13427a | ||
|
|
2049b1a370 | ||
|
|
43d4f511c5 | ||
|
|
f7f3518a04 | ||
|
|
d4aa1fe11b | ||
|
|
14046f9b14 | ||
|
|
be88162cde | ||
|
|
a192052d82 | ||
|
|
1dae2de835 | ||
|
|
0490043990 | ||
|
|
e094e362f4 | ||
|
|
8850a5673e | ||
|
|
cc9c681fa3 | ||
|
|
8f87dbe94d | ||
|
|
7cee3842b8 | ||
|
|
f1c3343929 | ||
|
|
63c8687166 | ||
|
|
b13f65674b | ||
|
|
e6ee939bd4 | ||
|
|
bde60130a8 | ||
|
|
795724f5ca | ||
|
|
44fee9eb0e | ||
|
|
cbcbb9644b | ||
|
|
d1a256b429 | ||
|
|
40bbc7e0d3 | ||
|
|
70f13a3775 | ||
|
|
189e64c486 | ||
|
|
1a1d3a1ecc | ||
|
|
288364f28d | ||
|
|
447bcf3a6b | ||
|
|
07d26adcd3 | ||
|
|
02214cf633 | ||
|
|
f904ff425e | ||
|
|
89370b6f5c | ||
|
|
4edd0e526c | ||
|
|
5d5ee69775 | ||
|
|
9125fcc712 | ||
|
|
17ef1d577d | ||
|
|
da27cd1593 | ||
|
|
aa6a4e7a59 | ||
|
|
8bf45fbafe | ||
|
|
e18be5cabe | ||
|
|
a78fa3db9d | ||
|
|
b81d7475e8 | ||
|
|
7ec6767418 | ||
|
|
30ec4329e0 | ||
|
|
3bae0d8fa2 | ||
|
|
7bd18306d4 | ||
|
|
1302eb9e1f | ||
|
|
f2ed0938ce | ||
|
|
4a84370b2f | ||
|
|
7d1b8864cc | ||
|
|
b18c62f671 | ||
|
|
cc38ca3f8d | ||
|
|
d739516056 | ||
|
|
a6c225eb45 | ||
|
|
5085b681e9 | ||
|
|
9e407f8f76 | ||
|
|
8545e7fc9f | ||
|
|
55a506975b | ||
|
|
b7dfe4c637 | ||
|
|
c4b05a3e93 | ||
|
|
38f2401f17 | ||
|
|
c8b041ab37 | ||
|
|
f52f47e1ff | ||
|
|
213ebe7cfd | ||
|
|
90e278cb97 | ||
|
|
58748c2be1 | ||
|
|
8e9cc58301 | ||
|
|
fb96216e46 | ||
|
|
5059fef3e6 | ||
|
|
564a04b45c | ||
|
|
edf7b3c0bf | ||
|
|
0e819eea4d | ||
|
|
f4ecc6fa64 | ||
|
|
fa918318b2 | ||
|
|
3f2303a086 | ||
|
|
ae6a54bca2 | ||
|
|
d6807233f2 | ||
|
|
3ae308426a | ||
|
|
8a98271680 | ||
|
|
0469641b9d | ||
|
|
4f6f38e16a | ||
|
|
5b6602544e | ||
|
|
efef7183ef | ||
|
|
a4772a5c1d | ||
|
|
26852f4a9d | ||
|
|
82776466b2 | ||
|
|
4837548349 | ||
|
|
b1f262d7d0 | ||
|
|
ddadb75bf4 | ||
|
|
91128d4f0a | ||
|
|
5dbde07b7b | ||
|
|
8a658a4290 | ||
|
|
fb8d2386b9 | ||
|
|
a2558171e1 | ||
|
|
f83835b4b6 | ||
|
|
a313c3e843 | ||
|
|
e7f51caedb | ||
|
|
4b194215dc | ||
|
|
af0fd83aec | ||
|
|
17abe6af32 | ||
|
|
804d5cc065 | ||
|
|
fd62a36041 | ||
|
|
14f68219cf | ||
|
|
7235e4fb92 | ||
|
|
e559798972 | ||
|
|
2dd5239726 | ||
|
|
d409bdf20f | ||
|
|
09791f5a82 | ||
|
|
6298d9d67c | ||
|
|
83391dba1b | ||
|
|
52a1570b3e | ||
|
|
8fe6442c47 | ||
|
|
1fef458c5f | ||
|
|
c104e7500d | ||
|
|
e0b56c83f9 | ||
|
|
d4ac50c4b1 | ||
|
|
f756731700 | ||
|
|
255f8b2aca | ||
|
|
ff58b16cf9 | ||
|
|
788969bcf8 | ||
|
|
ce054dc8dd | ||
|
|
94f555277c | ||
|
|
031cfb8277 | ||
|
|
c1b28cc4fe | ||
|
|
33973d5338 | ||
|
|
f800d672f3 | ||
|
|
98594c9902 | ||
|
|
57193ab2e4 | ||
|
|
f7cd4698db | ||
|
|
180a3fe5db | ||
|
|
4bb1970fef | ||
|
|
ab210fc5ab | ||
|
|
8edc83950c | ||
|
|
8e470f3af4 | ||
|
|
5588875c3f | ||
|
|
a2dd7a5c37 | ||
|
|
7536d9d14e | ||
|
|
9b60eda451 | ||
|
|
541ad1d2e2 | ||
|
|
ce7032c49b | ||
|
|
08ecf10ef3 | ||
|
|
ba502479e8 | ||
|
|
e564eaf75f | ||
|
|
a54d1acf45 | ||
|
|
e82ea89c23 | ||
|
|
06b58bc83b | ||
|
|
17c0734fb2 | ||
|
|
c045057ca0 | ||
|
|
b44f01f89c | ||
|
|
229a3e9855 | ||
|
|
e5192f55ee | ||
|
|
ee2f1efe78 | ||
|
|
8a91871de6 | ||
|
|
64a5074d58 | ||
|
|
7b7796b6fa | ||
|
|
f56eb62d8b | ||
|
|
78f03b7422 | ||
|
|
3aec7f9b02 | ||
|
|
2488d839ab | ||
|
|
2531cd42b6 | ||
|
|
5235bb2e79 | ||
|
|
3b6f3af38e | ||
|
|
15a15cbda5 | ||
|
|
7dbf7a19a1 | ||
|
|
e1fd3eb0f6 | ||
|
|
73280f85c3 | ||
|
|
b8e8981730 | ||
|
|
da787ae360 | ||
|
|
013b7420f6 | ||
|
|
c7fbc530da | ||
|
|
98ad1aa252 | ||
|
|
b01cf63237 | ||
|
|
883d976b5f | ||
|
|
ba4bfea0a4 | ||
|
|
fb7ad5cd73 | ||
|
|
5e931eac19 | ||
|
|
d54eab4052 | ||
|
|
c1b4533201 | ||
|
|
6771db22ae | ||
|
|
d8274204fb | ||
|
|
17c79f25f0 | ||
|
|
d4cbfc1359 | ||
|
|
e0af965d8b | ||
|
|
64d364a768 | ||
|
|
2caaa6fc92 | ||
|
|
fa075dc880 | ||
|
|
f6df07605d | ||
|
|
fc3883b0b6 | ||
|
|
5b2e5bc537 | ||
|
|
25953c6ae1 | ||
|
|
38f89c6976 | ||
|
|
49ea5c96e3 | ||
|
|
7692f201df | ||
|
|
09da8d741e | ||
|
|
b2351a213d | ||
|
|
2abceb1913 | ||
|
|
685810720e | ||
|
|
8e25926e91 | ||
|
|
68628bfd78 | ||
|
|
1dbb7c5740 | ||
|
|
9e3d1d5e68 | ||
|
|
9a70d697af | ||
|
|
a4116a1de7 | ||
|
|
6f721dd536 | ||
|
|
354b9ea396 | ||
|
|
3618831992 | ||
|
|
843a1589d7 | ||
|
|
fc6d5dcbdb | ||
|
|
6d709e1fc1 | ||
|
|
eb11b2c844 | ||
|
|
67e3ebf22f | ||
|
|
9b31746cdb | ||
|
|
ebad58f7e2 | ||
|
|
3c6fb7fc9e | ||
|
|
71aa3e86c6 | ||
|
|
2d8ad2e66a | ||
|
|
1f6f4eda32 | ||
|
|
769815ac0f | ||
|
|
e7888024a8 | ||
|
|
2007a79c45 | ||
|
|
d8286d1790 | ||
|
|
ca74639a74 | ||
|
|
a58a73bb55 | ||
|
|
16594feff1 | ||
|
|
cb46ff8735 | ||
|
|
6f079f13a8 | ||
|
|
a3a0d8f2ce | ||
|
|
5fe8ab11f1 | ||
|
|
b6eb43dd62 | ||
|
|
dc4419b5c5 | ||
|
|
24181c0242 | ||
|
|
db737df597 | ||
|
|
dd9acaaeb3 | ||
|
|
451a4dfe74 | ||
|
|
fb26ca5112 | ||
|
|
389bb96ae4 | ||
|
|
06e668cb90 | ||
|
|
297a3cb549 | ||
|
|
1d0467d4c3 | ||
|
|
9db9499267 | ||
|
|
deda4c985e | ||
|
|
06df6a6167 | ||
|
|
1718d4b1d7 | ||
|
|
16b71ca685 | ||
|
|
5d57b78fe9 | ||
|
|
36332abf35 | ||
|
|
dbbdc95c0b | ||
|
|
fb0be84e24 | ||
|
|
730e44d762 | ||
|
|
f4dd334b49 | ||
|
|
2ac146078f | ||
|
|
726bbf1f1f | ||
|
|
e753009fbc | ||
|
|
f39bc10e3b | ||
|
|
3629bcf057 | ||
|
|
743a3618ea | ||
|
|
77d738d973 | ||
|
|
a3b4ac79af | ||
|
|
c6dd04a7cc | ||
|
|
48cd554f46 | ||
|
|
31bbf207ef | ||
|
|
2b801f5bd4 | ||
|
|
790653dfde | ||
|
|
89be76bb6a | ||
|
|
704fc8d467 | ||
|
|
458e403a04 | ||
|
|
93c0ae19dd | ||
|
|
4e9b076408 | ||
|
|
ae7ec702a5 | ||
|
|
2cccbfd096 | ||
|
|
8f5e7f0c8f | ||
|
|
75d3c40f41 | ||
|
|
63582e44f3 | ||
|
|
4d4743d900 | ||
|
|
289d1e8fa6 | ||
|
|
968b9bf2e6 | ||
|
|
bab2e1391a | ||
|
|
663c9cbc34 | ||
|
|
3d916a685e | ||
|
|
a49fd9cc65 | ||
|
|
3192d89edf | ||
|
|
9edfcc40e9 | ||
|
|
68bdda1a4d | ||
|
|
bb7f2ff365 | ||
|
|
da51481620 | ||
|
|
c93f0d15e4 | ||
|
|
e0cb285993 | ||
|
|
a7d03f7c82 | ||
|
|
ea3bfa5e6e | ||
|
|
ec0b7e2dcc | ||
|
|
feb32600bb | ||
|
|
bc1978ec67 | ||
|
|
24cb379434 | ||
|
|
96be0c3377 | ||
|
|
c38e81c7f5 | ||
|
|
bb63d83f84 | ||
|
|
2915d74822 | ||
|
|
06d4f45e18 | ||
|
|
77dacfbb17 | ||
|
|
d8592d9a4f | ||
|
|
ea4a163187 | ||
|
|
fc92202e04 | ||
|
|
d8da56c6da | ||
|
|
c2b1ba4c5d | ||
|
|
04bb2d1398 | ||
|
|
ad60ba3a46 | ||
|
|
ea3eac51bd | ||
|
|
cacf537004 | ||
|
|
f4a86f5884 | ||
|
|
9f1b227fb8 | ||
|
|
6c79e456a1 | ||
|
|
0b980203fe | ||
|
|
73bf290f58 | ||
|
|
475bb4bb0b | ||
|
|
efadc691e7 | ||
|
|
4af36c69a7 | ||
|
|
bd107d2015 | ||
|
|
f007ca4227 | ||
|
|
50c4674dfa | ||
|
|
c12b870104 | ||
|
|
92aedb9365 | ||
|
|
368204115c | ||
|
|
971cdb44e2 | ||
|
|
16e4e61a0f | ||
|
|
4b36cc29a6 | ||
|
|
31ddd09087 | ||
|
|
68173de8bd | ||
|
|
960e3309bc | ||
|
|
2ee7589474 | ||
|
|
297992a4ae | ||
|
|
3f85230417 | ||
|
|
43c3158bc6 | ||
|
|
d95e9f3229 | ||
|
|
d581f9659a | ||
|
|
ea94f9234c | ||
|
|
c19468b04e | ||
|
|
c47afd8f4b | ||
|
|
afd5a3ba07 | ||
|
|
7dbad4c6d2 | ||
|
|
083a5042f6 | ||
|
|
351ca7c290 | ||
|
|
f907be2c14 | ||
|
|
6201f58c20 | ||
|
|
8fee764c73 | ||
|
|
3dd94f714d | ||
|
|
c716670be0 | ||
|
|
352066dde6 | ||
|
|
d6bd31a166 | ||
|
|
c0a4b139a2 | ||
|
|
6d60953f98 | ||
|
|
17ed06ee4d | ||
|
|
9427dac823 | ||
|
|
d74e082155 | ||
|
|
2e0013ec25 | ||
|
|
2493aa6529 | ||
|
|
85c89b16ef | ||
|
|
4f75925410 | ||
|
|
820c458f4c | ||
|
|
dc9d65524d | ||
|
|
1a4c788ed7 | ||
|
|
df8b9ac3b1 | ||
|
|
fcb1e40582 | ||
|
|
4973372612 | ||
|
|
3f6c085e8a | ||
|
|
040e3097b5 | ||
|
|
feef96ed16 | ||
|
|
4f722899df | ||
|
|
166a918b43 | ||
|
|
1b4c860f63 | ||
|
|
3b6945b2db | ||
|
|
4174e23c04 | ||
|
|
3993175e45 | ||
|
|
60c7ffc981 | ||
|
|
33de9dde4f | ||
|
|
6b85220fa4 | ||
|
|
1cbae7d41c | ||
|
|
b3a4b09f5b | ||
|
|
c8b26735c8 | ||
|
|
907e092abc | ||
|
|
3073495a06 | ||
|
|
16c411e436 | ||
|
|
2d841870a8 | ||
|
|
7f8ce7f5c0 | ||
|
|
07b5f13474 | ||
|
|
f4aac599ca | ||
|
|
2d63f0ba62 | ||
|
|
1d41682715 | ||
|
|
705765d113 | ||
|
|
16a97ed65d | ||
|
|
ecf532af0e | ||
|
|
ce600df98a | ||
|
|
b4a6a34705 | ||
|
|
a7e51b416b | ||
|
|
71f786f146 | ||
|
|
848c6fdc56 | ||
|
|
ead9ee2d13 | ||
|
|
4861d1f78d | ||
|
|
5c76c9d3ac | ||
|
|
699cbfb7c1 | ||
|
|
319d56be7e | ||
|
|
11d3678984 | ||
|
|
f55f150896 | ||
|
|
e1f1b5d18b | ||
|
|
f0fe454145 | ||
|
|
4026f6c188 | ||
|
|
29cfb8c177 | ||
|
|
a092ee9ff9 | ||
|
|
6c3cbee35c | ||
|
|
4b23db28b3 | ||
|
|
646f83642a | ||
|
|
d9bfba1e62 | ||
|
|
f1bb7747ae | ||
|
|
d2eb3e5ecf | ||
|
|
b0b1cf4f01 | ||
|
|
d91ea41f64 | ||
|
|
5fed429827 | ||
|
|
9a569bfb8d | ||
|
|
e2c41ffa5a | ||
|
|
13e4ed8515 | ||
|
|
c66fe11e16 | ||
|
|
75d0eb32be | ||
|
|
c44b9f91cd | ||
|
|
82da252cf0 | ||
|
|
ce72b67cef | ||
|
|
32bc4f3c48 | ||
|
|
2a94d16834 | ||
|
|
da7baa5e39 | ||
|
|
ed1125b66d | ||
|
|
366bc750da | ||
|
|
04a56b7aff | ||
|
|
2f697216ee | ||
|
|
769dab43a2 | ||
|
|
98a05997ae | ||
|
|
07dee528c5 | ||
|
|
7ddbf40715 | ||
|
|
aea22bf793 | ||
|
|
72889be87f | ||
|
|
86db64904b | ||
|
|
12f16bbebd | ||
|
|
4ec40eec9a | ||
|
|
13380e3d0e | ||
|
|
9132b6a296 | ||
|
|
bcb5aafdbe | ||
|
|
3e8bd9b0de | ||
|
|
b842b9422d | ||
|
|
9db15c7cf4 | ||
|
|
ba47b47e24 | ||
|
|
b566563dc5 | ||
|
|
2a856e2a76 | ||
|
|
ce7d60cb6e | ||
|
|
26ba7404bf | ||
|
|
b4af9cdb7a | ||
|
|
db53ec975f | ||
|
|
1ff5049d03 | ||
|
|
41f25d3129 | ||
|
|
4628ba26b4 | ||
|
|
20027e53ff | ||
|
|
d4a054bfed | ||
|
|
c623e8d235 | ||
|
|
998fe3a515 | ||
|
|
73be3df28c | ||
|
|
eb65535f45 | ||
|
|
fa6cdbbe61 | ||
|
|
462ec9cfb1 | ||
|
|
9950689a11 | ||
|
|
71a6a5bf77 | ||
|
|
22d4cbd144 | ||
|
|
7b618b2317 | ||
|
|
d8477c71b1 | ||
|
|
45fae9da8a | ||
|
|
2f742fd3fe | ||
|
|
82180bffd2 | ||
|
|
ac82187b85 | ||
|
|
8d38c6bac9 | ||
|
|
9afef800c0 | ||
|
|
eaf27f36af | ||
|
|
4efe6cfb26 | ||
|
|
3e7f852472 |
@@ -1 +1,4 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// AssignerModule is the asker the assigner's compose asks name.
|
||||
const AssignerModule = "mesh-delivery-assigner"
|
||||
|
||||
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
|
||||
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
|
||||
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
|
||||
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
|
||||
// second ask builds nothing.
|
||||
type composeAsks struct {
|
||||
ctx context.Context
|
||||
asker buildrun.Asker
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
asks map[string]*composeAsk
|
||||
}
|
||||
|
||||
type composeAsk struct {
|
||||
id string
|
||||
cancel context.CancelFunc
|
||||
ran *assign.Ran
|
||||
}
|
||||
|
||||
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
|
||||
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
|
||||
}
|
||||
|
||||
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
|
||||
// A job its layout refuses ends at once, failed with why: never silent.
|
||||
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if k, ok := a.asks[key]; ok {
|
||||
return k.ran, k.id
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
|
||||
At: a.now(), Compose: &job}
|
||||
ctx, cancel := context.WithCancel(a.ctx)
|
||||
k := &composeAsk{id: ask.ID, cancel: cancel}
|
||||
a.asks[key] = k
|
||||
if err := a.asker.Put(ctx, ask); err != nil {
|
||||
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
|
||||
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
|
||||
cancel()
|
||||
return k.ran, k.id
|
||||
}
|
||||
go func() {
|
||||
step, err := a.asker.Wait(ctx, ask)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
ran := assign.RanOf(step)
|
||||
if err != nil {
|
||||
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
|
||||
}
|
||||
a.mu.Lock()
|
||||
k.ran = &ran
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}()
|
||||
return nil, k.id
|
||||
}
|
||||
|
||||
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
|
||||
func (a *composeAsks) sweep(seen map[string]bool) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for key, k := range a.asks {
|
||||
if !seen[key] {
|
||||
k.cancel()
|
||||
delete(a.asks, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
|
||||
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
|
||||
// asks. The keys it used are marked in seen.
|
||||
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
|
||||
waiting := func(words string) delivery.CompositionVerdict {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
|
||||
}
|
||||
head, err := assign.HeadJob(m, ch)
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
|
||||
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
|
||||
}
|
||||
var pending []string
|
||||
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
|
||||
seen[key] = true
|
||||
headRan, id := p.asks.get(key, head)
|
||||
if headRan == nil {
|
||||
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
|
||||
}
|
||||
var readers []assign.Reader
|
||||
var notJudged []string
|
||||
for _, module := range sortedNames(assign.ReaderModules) {
|
||||
repository := assign.ReaderModules[module]
|
||||
var commits []string
|
||||
member := false
|
||||
if h := ch.Members[repository]; h != "" {
|
||||
commits, member = []string{h}, true
|
||||
} else {
|
||||
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
|
||||
"commit on record, and the group holds no member of "+repository)
|
||||
continue
|
||||
}
|
||||
for _, commit := range commits {
|
||||
job, err := assign.ReaderJob(ch, module, repository, commit)
|
||||
if err != nil {
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
|
||||
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
|
||||
continue
|
||||
}
|
||||
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
|
||||
seen[rkey] = true
|
||||
ran, rid := p.asks.get(rkey, job)
|
||||
if ran == nil {
|
||||
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
|
||||
continue
|
||||
}
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
|
||||
}
|
||||
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
|
||||
v.Warnings = append(v.Warnings, notJudged...)
|
||||
return v
|
||||
}
|
||||
|
||||
func sortedNames(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
-180
@@ -1,180 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
|
||||
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
|
||||
type holder struct {
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
}
|
||||
|
||||
func (h *holder) taken() []delivery.StepAsk {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), h.asks...)
|
||||
}
|
||||
|
||||
func (b *bed) hold(t *testing.T) *holder {
|
||||
t.Helper()
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
|
||||
h := &holder{}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
a, err := delivery.DecodeStepAsk(m.Data())
|
||||
if err != nil || a.Compose == nil {
|
||||
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
|
||||
m.Ack()
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.asks = append(h.asks, a)
|
||||
h.mu.Unlock()
|
||||
req, err := composer.Decode(a.Compose.Request)
|
||||
if err == nil {
|
||||
err = composer.Fill(&req, func(digest string) ([]byte, error) {
|
||||
key, _ := delivery.ManifestsKey(digest)
|
||||
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
man, err := delivery.DecodeManifest(e.Value())
|
||||
return []byte(man.JSON), err
|
||||
})
|
||||
}
|
||||
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else if v, err := composer.Run(req); err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else {
|
||||
vr, _ := json.Marshal(v)
|
||||
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
|
||||
}
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
t.Errorf("the step cannot be written: %v", err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
if _, err := steps.Put(b.ctx, key, raw); err != nil {
|
||||
t.Errorf("the step %s: %v", key, err)
|
||||
}
|
||||
m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(cc.Stop)
|
||||
return h
|
||||
}
|
||||
|
||||
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
|
||||
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
|
||||
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// head's composer was built with, and asks nothing more on a later pass of the same heads.
|
||||
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
running := strings.Repeat("3", 40)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
|
||||
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
b.start(t)
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
// No holder takes the asks yet: the composition waits, naming them.
|
||||
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
|
||||
}
|
||||
h := b.hold(t)
|
||||
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
|
||||
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14", v)
|
||||
}
|
||||
asks := h.taken()
|
||||
if len(asks) != 2 {
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
|
||||
}
|
||||
byRepo := map[string]delivery.StepAsk{}
|
||||
for _, a := range asks {
|
||||
byRepo[a.Compose.Fetch["repository"].(string)] = a
|
||||
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
|
||||
t.Errorf("ask = %+v", a)
|
||||
}
|
||||
}
|
||||
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
|
||||
t.Errorf("the head's ask = %+v", a.Compose)
|
||||
}
|
||||
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running controller's ask = %+v", a.Compose)
|
||||
}
|
||||
// A later pass of the same heads (woken by any change) asks nothing more.
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := len(h.taken()); n != 2 {
|
||||
t.Errorf("a later pass of the same heads asked again: %d asks", n)
|
||||
}
|
||||
}
|
||||
|
||||
// verdict waits for the composition under key to satisfy ok, and answers it.
|
||||
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
|
||||
t.Helper()
|
||||
var last delivery.CompositionVerdict
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
|
||||
return last
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
|
||||
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
|
||||
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
|
||||
// where they part. The assigner composes nothing and never writes these.
|
||||
|
||||
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
|
||||
// it was refused or cannot be read.
|
||||
type publishedRow struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Unread string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
|
||||
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
|
||||
// A value or key that does not read is named with why, never left out.
|
||||
func published(values map[string][]byte) map[string][]publishedRow {
|
||||
byNode := map[string]map[string][]byte{}
|
||||
for key, raw := range values {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
if byNode[node] == nil {
|
||||
byNode[node] = map[string][]byte{}
|
||||
}
|
||||
byNode[node][key] = raw
|
||||
}
|
||||
out := map[string][]publishedRow{}
|
||||
for node, keys := range byNode {
|
||||
var rows []publishedRow
|
||||
for key, raw := range keys {
|
||||
row := publishedRow{Key: key}
|
||||
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
row.Unread = err.Error()
|
||||
case isNode:
|
||||
row.Module = "node"
|
||||
default:
|
||||
row.Module, row.FP = module, fp
|
||||
}
|
||||
if err == nil {
|
||||
c, derr := delivery.DecodeComposed(raw)
|
||||
switch {
|
||||
case derr != nil:
|
||||
row.Unread = derr.Error()
|
||||
case c.Refused != "":
|
||||
row.Refused = c.Refused
|
||||
default:
|
||||
if d, err := delivery.ComposedDigestOf(c); err != nil {
|
||||
row.Unread = err.Error()
|
||||
} else {
|
||||
row.Digest = d
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
|
||||
out[node] = rows
|
||||
}
|
||||
return out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,8 +14,9 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"slices"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -23,12 +24,23 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv(process.BrokerFileEnv) == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
@@ -40,85 +52,139 @@ func main() {
|
||||
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
}
|
||||
|
||||
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
|
||||
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
|
||||
var readFamilies = []string{delivery.ComposedRead}
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
|
||||
// (ADR 0567 §1), named after the assigner as every module state is.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket}
|
||||
|
||||
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
|
||||
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
|
||||
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller.
|
||||
// (and the node agent from release 6), its differences by the view, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
|
||||
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
{Name: delivery.PlacementCandidatesState, Readable: true},
|
||||
{Name: delivery.PlacementPulsesState, Readable: true},
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
{Name: health.FindingsState, Readable: true},
|
||||
{Name: delivery.AssignerPoliciesState, Readable: true},
|
||||
{Name: delivery.SeatHoldersState, Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
type buckets map[string]jetstream.KeyValue
|
||||
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||
// without it would write wants from half the mesh.
|
||||
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
|
||||
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
out := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.SeatHoldersBucket && err != nil {
|
||||
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
|
||||
// holds which seat here, and the controller's seats still answer.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == provisions.ProvisionsBucket && err != nil {
|
||||
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
|
||||
// until then no want waits for a provision, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.PoliciesBucket && err != nil {
|
||||
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
|
||||
// person's policy is read, so the controller's beside each composition holds, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.FindingsBucket && err != nil {
|
||||
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
|
||||
// router and kept as no finding.
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
|
||||
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
|
||||
// a record module is asked of a person, as before.
|
||||
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
out[name] = kv
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's
|
||||
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
|
||||
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||
}
|
||||
account, err := process.ReadAccount(path)
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(account.Node + "." + account.Module),
|
||||
nats.UserInfo(account.User, account.Password),
|
||||
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(account.URL, opts...)
|
||||
conn, js, err := account.Dial()
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js)
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,31 +198,64 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := said.Keeps(keptStates...); err != nil {
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
thisNode = account.Node
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
||||
findings := findingsOf(ctx, kvs, logger.Printf)
|
||||
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger)
|
||||
}
|
||||
|
||||
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
|
||||
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
|
||||
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
|
||||
var f assign.Findings
|
||||
if kv := kvs[assign.FindingsBucket]; kv != nil {
|
||||
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
|
||||
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
|
||||
f.Keys = func() ([]string, error) {
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
|
||||
var thisNode string
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
delivery.AssignerCompositionsBucket,
|
||||
delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerWantsBucket,
|
||||
assign.PoliciesBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SeatHoldersBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
}
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -165,13 +264,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
default:
|
||||
}
|
||||
}
|
||||
errs := make(chan error, len(watched)+1)
|
||||
settled := make(chan struct{}, len(watched)+1)
|
||||
errs := make(chan error, len(watched)+2)
|
||||
settled := make(chan struct{}, len(watched)+2)
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||
}
|
||||
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
|
||||
optional := name == delivery.AssignerPutsBucket
|
||||
go func() {
|
||||
defer kw.Stop()
|
||||
for {
|
||||
@@ -181,28 +282,73 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
return
|
||||
case e, ok := <-kw.Updates():
|
||||
if !ok {
|
||||
if optional {
|
||||
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
|
||||
return
|
||||
}
|
||||
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
settled <- struct{}{}
|
||||
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
|
||||
settled <- struct{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
var value []byte
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
w.put(name, e.Key(), value)
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
wake()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
|
||||
// stops the puts, never the wants.
|
||||
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
}
|
||||
}
|
||||
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
|
||||
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
|
||||
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
|
||||
// further, without a restart.
|
||||
var triedPuts time.Time
|
||||
reopenPuts := func(p *passer) {
|
||||
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
|
||||
return
|
||||
}
|
||||
triedPuts = time.Now()
|
||||
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket] = kv
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && startsWithout(name) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
if name == delivery.HaveBucket || name == delivery.SourceBucket {
|
||||
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
|
||||
opts = append(opts, jetstream.IncludeHistory())
|
||||
}
|
||||
if err := watch(name, opts...); err != nil {
|
||||
return err
|
||||
@@ -211,7 +357,37 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
for range len(watched) + 1 {
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
|
||||
// module is placed.
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets)+2)
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
}
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
}
|
||||
for range started + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -220,8 +396,16 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -230,12 +414,37 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
|
||||
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
|
||||
// bus now holds, and says the seat waits while one is missing.
|
||||
var unbound []string
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] != nil || js == nil {
|
||||
continue
|
||||
}
|
||||
if kv, err := js.KeyValue(ctx, name); err == nil {
|
||||
if err := takePlacement(appeared{name, kv}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
unbound = append(unbound, name)
|
||||
}
|
||||
p.sayWaits(unbound)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
|
||||
@@ -1,33 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reads []string
|
||||
for _, r := range m.Reads {
|
||||
if slices.Contains(readFamilies, r) {
|
||||
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
|
||||
}
|
||||
reads = append(reads, strings.Replace(r, ".", "_", 1))
|
||||
}
|
||||
for _, l := range readFamilies {
|
||||
if !slices.Contains(m.Reads, l) {
|
||||
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
for _, s := range m.State {
|
||||
states = append(states, m.Module+"_"+s.Name)
|
||||
@@ -48,12 +70,116 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
func TestNoCredential(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
}
|
||||
}
|
||||
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
|
||||
// keep the first pass from being made: the wants are written all the same.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
wrote := make(chan string, 8)
|
||||
kvs := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[delivery.PlacementBucket].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
t.Errorf("the first pass wrote %s", key)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
// closed; a put on it is said on wrote.
|
||||
type watchKV struct {
|
||||
*memKV
|
||||
entries []kvEntry
|
||||
closed bool
|
||||
wrote chan string
|
||||
}
|
||||
|
||||
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
|
||||
if w.closed {
|
||||
close(ch)
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
for _, e := range w.entries {
|
||||
ch <- e
|
||||
}
|
||||
ch <- nil
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
|
||||
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if w.wrote != nil {
|
||||
w.wrote <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
|
||||
|
||||
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
|
||||
func (k keyWatcher) Stop() error { return nil }
|
||||
|
||||
type kvEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -25,14 +29,45 @@ type passer struct {
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
|
||||
failedProvisions map[string]bool
|
||||
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
|
||||
missing, missingSaid map[string]bool
|
||||
// unplaced are the nodes whose unplaced finding this process wrote.
|
||||
unplaced map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||
neither map[string]neitherOne
|
||||
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||
acts *putActs
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
|
||||
// provider is down, which holds no want): said once per words, and once more when it is reached again.
|
||||
unreached map[string]string
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
|
||||
// (clearStaleCalls): the assigner makes no such call now (issue 774).
|
||||
callsCleared bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// convertedAll says every node in the form before was converted to its node and module keys (convertPlacement).
|
||||
convertedAll bool
|
||||
// saidWaits is the last said of the seat's wait for its states (sayWaits).
|
||||
saidWaits string
|
||||
intended map[string]map[string]map[string]bool
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -43,25 +78,27 @@ func (p *passer) clock() time.Time {
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
p.placementSeat(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
@@ -71,7 +108,7 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
for name, mod := range trunk {
|
||||
@@ -90,6 +127,16 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
provisionsOf := p.provisionsOf()
|
||||
missing := map[string]bool{}
|
||||
defer func() { p.provisionsMissing(ctx, missing) }()
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -104,15 +151,21 @@ func (p *passer) wants(ctx context.Context) {
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(),
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
key, err := delivery.PartGrantKey(name, fp)
|
||||
return err == nil && p.w.has(grants, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
@@ -121,16 +174,17 @@ func (p *passer) wants(ctx context.Context) {
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
}}
|
||||
},
|
||||
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.PartGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
@@ -143,7 +197,17 @@ func (p *passer) wants(ctx context.Context) {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
for _, node := range m.Nodes {
|
||||
for _, prov := range m.MissingProvisions(node) {
|
||||
missing[node+"."+name+"."+prov] = true
|
||||
}
|
||||
}
|
||||
p.yeses(ctx, name, m, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
p.sayUnreached(name, m.Nodes, out)
|
||||
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
@@ -182,6 +246,116 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
|
||||
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
|
||||
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
|
||||
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
|
||||
if p.noWant == nil {
|
||||
p.noWant = map[string]string{}
|
||||
}
|
||||
assigned := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
assigned[node] = true
|
||||
key := node + "." + module
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = h.Reason()
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
case reason != "" && reason != last:
|
||||
p.logf("%s on %s gets no want: %s", module, node, reason)
|
||||
p.noWant[key] = reason
|
||||
case reason == "" && said && written && !w.Removed:
|
||||
p.logf("%s on %s has a want", module, node)
|
||||
delete(p.noWant, key)
|
||||
}
|
||||
}
|
||||
for key := range p.noWant {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
|
||||
delete(p.noWant, key) // no longer assigned there: its removed want says so
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
|
||||
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
|
||||
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
|
||||
if p.unreached == nil {
|
||||
p.unreached = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
key := node + "." + module
|
||||
why, last := out.Unreached[node], p.unreached[key]
|
||||
switch {
|
||||
case why != "" && why != last:
|
||||
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
|
||||
p.unreached[key] = why
|
||||
case why == "" && last != "":
|
||||
p.logf("%s on %s is reached: its providers run healthy", module, node)
|
||||
delete(p.unreached, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
|
||||
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
|
||||
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
|
||||
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
|
||||
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
|
||||
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
|
||||
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
|
||||
if p.nodeFailed == nil {
|
||||
p.nodeFailed = map[string]string{}
|
||||
}
|
||||
look := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
look[node] = true
|
||||
}
|
||||
for key := range p.nodeFailed {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
if first {
|
||||
for node := range known.Online {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(look)) {
|
||||
key := node + "." + module
|
||||
f, failing := out.NodeFailed[node]
|
||||
last, said := p.nodeFailed[key]
|
||||
switch {
|
||||
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
|
||||
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
|
||||
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
|
||||
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
|
||||
case !failing && (said || first):
|
||||
if said {
|
||||
p.logf("%s on %s no longer fails its health check alone", module, node)
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
|
||||
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
delete(p.nodeFailed, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
@@ -195,6 +369,48 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
@@ -211,7 +427,9 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
@@ -231,73 +449,30 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||
func (p *passer) compositions(ctx context.Context) {
|
||||
if p.sdk == "" {
|
||||
p.sdk = sdkversion.Version()
|
||||
}
|
||||
written := p.w.read(delivery.AssignerCompositionsBucket)
|
||||
trunk := p.w.trunk()
|
||||
var mesh *assign.Mesh
|
||||
seen := map[string]bool{}
|
||||
if p.asks != nil {
|
||||
defer p.asks.sweep(seen)
|
||||
}
|
||||
for key, value := range p.w.read(delivery.CandidatesBucket) {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err)
|
||||
continue
|
||||
}
|
||||
out, err := delivery.CompositionKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s names no composition key: %v", key, err)
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk)
|
||||
var v delivery.CompositionVerdict
|
||||
if waits != "" {
|
||||
v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits}
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
if mesh == nil {
|
||||
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
|
||||
mesh = &m
|
||||
}
|
||||
if ch.SDKHead != "" && p.asks != nil {
|
||||
// A group with a mesh-sdk member is composed by the composer built at its head (ADR 0482 clauses 3 to 5).
|
||||
v = p.composeAtHead(ctx, *mesh, ch, seen, began)
|
||||
} else {
|
||||
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||
}
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||
before.Took, before.V = v.Took, 0
|
||||
if delivery.SameCompositionVerdict(before, v) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) })
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,6 +483,11 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
p.logf("%s %s is not written: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
// A value the bucket already holds is not put again: every put is a change its watchers hear, and a want put unchanged
|
||||
// made its node compose again (issue 774, 2026-10-31: novox composed every 8 s, two asks of the vault each time).
|
||||
if held := p.w.value(bucket, key); held != nil && bytes.Equal(held, value) {
|
||||
return
|
||||
}
|
||||
if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be written: %v", bucket, key, err)
|
||||
return
|
||||
@@ -359,6 +539,106 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||
}
|
||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
|
||||
// set back from it (issue 1906, note 28).
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
|
||||
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
|
||||
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
|
||||
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
|
||||
// nothing taken away.
|
||||
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||
k, err := delivery.ParseAnyAskKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold, delivery.AskKindRestore:
|
||||
m, on := trunk[k.Module]
|
||||
return !on || (m.FP2 != "" && m.FP2 != k.FP)
|
||||
case delivery.AskKindRecord:
|
||||
m, on := trunk[k.Module]
|
||||
if !on {
|
||||
return true
|
||||
}
|
||||
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
|
||||
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
|
||||
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
|
||||
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||
// trunk's, its want's, its have's), and only when every one of them is read.
|
||||
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
|
||||
d, ok := p.w.composed(k.Node, k.Module, fp)
|
||||
if !ok || d.Digest == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
|
||||
// names, each once.
|
||||
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
|
||||
var out []string
|
||||
add := func(fp string) {
|
||||
if fp != "" && !slices.Contains(out, fp) {
|
||||
out = append(out, fp)
|
||||
}
|
||||
}
|
||||
add(trunkFP)
|
||||
if key, err := delivery.WantKey(node, module); err == nil {
|
||||
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
|
||||
add(w.FP)
|
||||
}
|
||||
}
|
||||
if key, err := delivery.HaveKey(node, module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
|
||||
add(h.FP)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
|
||||
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
|
||||
// read at all (no module), nothing is taken away.
|
||||
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
|
||||
if len(trunk) == 0 {
|
||||
return
|
||||
}
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if p.staleAsk(key, trunk) {
|
||||
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
|
||||
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
|
||||
k, kerr := delivery.ParseAnyAskKey(key)
|
||||
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
|
||||
if held && !p.clearHeld(k.Module, k.FP) {
|
||||
continue
|
||||
}
|
||||
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
|
||||
// raised again if the trunk returns to the fingerprint.
|
||||
if held {
|
||||
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||
@@ -433,16 +713,18 @@ func (p *passer) answeredWith(key, option string) bool {
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
func (p *passer) clearHeld(module, fp string) {
|
||||
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
|
||||
func (p *passer) clearHeld(module, fp string) bool {
|
||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||
return
|
||||
return true
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
p.said[c.Key+".cleared"] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||
@@ -534,3 +816,306 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
|
||||
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
|
||||
// want naming the release, with no ask (ADR 0601 §1).
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if m.Self == nil || !m.Self(node) {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes, release := "", ""
|
||||
if had && was.FP == w.FP {
|
||||
yes, release = was.Yes, was.Release
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
switch {
|
||||
case m.Allowed != nil && m.Allowed(node, h.Waits):
|
||||
yes, release = h.Waits, ""
|
||||
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
|
||||
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
|
||||
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
|
||||
if r, ok := m.Release(w.FP); ok {
|
||||
yes, release = h.Waits, r
|
||||
break
|
||||
}
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
default:
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes == "" {
|
||||
release = ""
|
||||
}
|
||||
if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chosenOf is a person's choice for module, nil when none.
|
||||
func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy {
|
||||
if c, ok := chosen[module]; ok {
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
|
||||
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
|
||||
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
|
||||
// needs-operator), deleted once it is no longer failed.
|
||||
func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision {
|
||||
if p.kvs[provisions.ProvisionsBucket] == nil {
|
||||
return func(string) func(string) map[string]provisions.Provision { return nil }
|
||||
}
|
||||
by := map[string]map[string]map[string]provisions.Provision{} // module, node, key
|
||||
failed := map[string]provisions.Provision{}
|
||||
for key, raw := range p.w.read(provisions.ProvisionsBucket) {
|
||||
pr, err := provisions.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
node, module := pr.Consumer.Node, pr.Consumer.Module
|
||||
if by[module] == nil {
|
||||
by[module] = map[string]map[string]provisions.Provision{}
|
||||
}
|
||||
if by[module][node] == nil {
|
||||
by[module][node] = map[string]provisions.Provision{}
|
||||
}
|
||||
by[module][node][key] = pr
|
||||
if pr.State == provisions.Failed {
|
||||
failed[key] = pr
|
||||
}
|
||||
}
|
||||
p.provisionFindings(failed)
|
||||
return func(module string) func(string) map[string]provisions.Provision {
|
||||
return func(node string) map[string]provisions.Provision { return by[module][node] }
|
||||
}
|
||||
}
|
||||
|
||||
// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed.
|
||||
func (p *passer) provisionFindings(failed map[string]provisions.Provision) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now
|
||||
if p.now != nil {
|
||||
now = p.now
|
||||
}
|
||||
for key, pr := range failed {
|
||||
if p.failedProvisions[key] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionFailedFinding(key, pr, now())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionFailedKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the failed provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.failedProvisions == nil {
|
||||
p.failedProvisions = map[string]bool{}
|
||||
}
|
||||
p.failedProvisions[key] = true
|
||||
}
|
||||
for key := range p.failedProvisions {
|
||||
if _, still := failed[key]; still {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil {
|
||||
p.logf("the failed provision %s is not cleared: %v", key, err)
|
||||
continue
|
||||
}
|
||||
delete(p.failedProvisions, key)
|
||||
}
|
||||
}
|
||||
|
||||
// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
|
||||
// when its manifest names none (ADR 0564 clause 13 (d)).
|
||||
var MissingBound = delivery.DefaultDeliveryBound
|
||||
|
||||
// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
|
||||
func missingPulse(key string) string { return "provision-missing." + key }
|
||||
|
||||
// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
|
||||
// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
|
||||
// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
|
||||
// the watch, is the pass that writes it: no timer.
|
||||
func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
|
||||
pulses := p.kvs[delivery.PlacementPulsesBucket]
|
||||
if p.missing == nil {
|
||||
p.missing = map[string]bool{}
|
||||
}
|
||||
for key := range missing {
|
||||
switch {
|
||||
case !p.missing[key]:
|
||||
p.missing[key] = true
|
||||
if pulses != nil {
|
||||
p.pulse(ctx, missingPulse(key), MissingBound)
|
||||
}
|
||||
case pulses == nil || !p.pulsing(missingPulse(key)):
|
||||
if p.missingSaid[key] || p.findings.Put == nil {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionMissingKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the missing provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.missingSaid == nil {
|
||||
p.missingSaid = map[string]bool{}
|
||||
}
|
||||
p.missingSaid[key] = true
|
||||
}
|
||||
}
|
||||
for key := range p.missing {
|
||||
if missing[key] {
|
||||
continue
|
||||
}
|
||||
delete(p.missing, key)
|
||||
if pulses != nil && p.pulsing(missingPulse(key)) {
|
||||
if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil)
|
||||
}
|
||||
}
|
||||
if p.missingSaid[key] && p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
|
||||
p.logf("the missing provision %s is not cleared: %v", key, err)
|
||||
p.missing[key] = true
|
||||
continue
|
||||
}
|
||||
delete(p.missingSaid, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unplacedFindings writes, from the lease write, the finding `machine.<node>.unplaced` for each node that runs but no
|
||||
// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed.
|
||||
func (p *passer) unplacedFindings() {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := []string(p.w.unplacedNodes())
|
||||
is := map[string]bool{}
|
||||
for _, node := range now {
|
||||
is[node] = true
|
||||
if p.unplaced[node] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.UnplacedFinding(node, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.UnplacedKey(node), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the unplaced node %s is not said: %v", node, err)
|
||||
continue
|
||||
}
|
||||
if p.unplaced == nil {
|
||||
p.unplaced = map[string]bool{}
|
||||
}
|
||||
p.unplaced[node] = true
|
||||
}
|
||||
for node := range p.unplaced {
|
||||
if is[node] {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil {
|
||||
delete(p.unplaced, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per
|
||||
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
|
||||
// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no
|
||||
// timer writes it.
|
||||
func (p *passer) seatHolders(ctx context.Context) {
|
||||
kv := p.kvs[delivery.SeatHoldersBucket]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
placements := p.w.placements()
|
||||
if len(placements) == 0 {
|
||||
return // no placement read yet: nothing is known of who holds what, and nothing is deleted
|
||||
}
|
||||
want := delivery.SeatHoldersOf(placements)
|
||||
have := p.w.read(delivery.SeatHoldersBucket)
|
||||
for key, sh := range want {
|
||||
value, err := json.Marshal(sh)
|
||||
if err != nil || string(have[key]) == string(value) {
|
||||
continue
|
||||
}
|
||||
p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil })
|
||||
}
|
||||
for key := range have {
|
||||
if _, ok := want[key]; ok {
|
||||
continue
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil {
|
||||
p.logf("the seat holders %s are not deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.SeatHoldersBucket, key, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name
|
||||
// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from.
|
||||
func offered(manifests map[string]catalogue.Manifest) func(string) bool {
|
||||
names := map[string]bool{}
|
||||
add := func(m catalogue.Manifest) {
|
||||
for _, o := range m.Offers() {
|
||||
names[o] = true
|
||||
}
|
||||
}
|
||||
for _, m := range manifests {
|
||||
add(m)
|
||||
}
|
||||
for _, module := range delivery.ProvidedModules() {
|
||||
if m, _, err := delivery.ProvidedManifest(module); err == nil {
|
||||
add(m)
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return names[provision] }
|
||||
}
|
||||
|
||||
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
|
||||
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
|
||||
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
|
||||
return func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) {
|
||||
if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -11,8 +14,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -21,9 +24,16 @@ import (
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
puts int // how many puts reached the bucket
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
m.puts++
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
@@ -39,6 +49,34 @@ func newPasser(kvs buckets, at time.Time) *passer {
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
|
||||
// (research 072 row 37).
|
||||
func placeOn(t *testing.T, w *world, node string, modules ...string) {
|
||||
t.Helper()
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
|
||||
func composes(t *testing.T, w *world, node, digest string) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
@@ -73,6 +111,14 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
// ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides
|
||||
// itself (mesh-addressing, mesh-wireguard's); nothing else.
|
||||
off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}})
|
||||
for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} {
|
||||
if off(provision) != want {
|
||||
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
|
||||
}
|
||||
}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
@@ -101,12 +147,248 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
type memEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (e memEntry) Value() []byte { return e.value }
|
||||
func (e memEntry) Revision() uint64 { return e.revision }
|
||||
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := m.values[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return memEntry{value: v, revision: uint64(len(v))}, nil
|
||||
}
|
||||
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if uint64(len(m.values[key])) != revision {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
m.values[key] = value
|
||||
return uint64(len(value)), nil
|
||||
}
|
||||
|
||||
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
if s[state] == nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
func (s memStore) put(state, key string, value []byte) error {
|
||||
s[state].values[key] = value
|
||||
return nil
|
||||
}
|
||||
func (s memStore) keys(state string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range s[state].values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
|
||||
type fakeBus struct {
|
||||
calls chan string
|
||||
events []string
|
||||
}
|
||||
|
||||
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
b.calls <- fmt.Sprintf("%s %v", key, body)
|
||||
return json.RawMessage(`{"ok":true,"output":""}`), nil
|
||||
}
|
||||
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(node+"/"+key, body)
|
||||
}
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
|
||||
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
|
||||
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
|
||||
// gives the want back to the rules. Each step is said as the event put.
|
||||
func putOfTheBus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
|
||||
puts, asksKV, wantsKV := mem(), mem(), mem()
|
||||
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
|
||||
world := func(bucket, key string, v any) {
|
||||
var data []byte
|
||||
switch x := v.(type) {
|
||||
case []byte:
|
||||
data = x
|
||||
default:
|
||||
data, _ = json.Marshal(x)
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "nats")
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
bkey, _ := delivery.BuilderBuildKey("nats", fp)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
|
||||
world(delivery.BuilderBuildsBucket, bkey, built)
|
||||
hkey, _ := delivery.HaveKey("novox", "nats")
|
||||
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
|
||||
builds := mem()
|
||||
builds.values[bkey] = built
|
||||
modules := mem()
|
||||
modules.values["nats"] = mod
|
||||
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
|
||||
s.now = func() time.Time { return now }
|
||||
was := answerWithin
|
||||
answerWithin = 0
|
||||
defer func() { answerWithin = was }()
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a prefix of a fingerprint the builder never built was taken")
|
||||
}
|
||||
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
|
||||
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
|
||||
}
|
||||
if got, err := s.version("nats", "newest"); err != nil || got != fp {
|
||||
t.Errorf("newest reads %q, %v", got, err)
|
||||
}
|
||||
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
|
||||
t.Fatalf("put answered %+v", a)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("novox", "nats")
|
||||
step := func() delivery.Put {
|
||||
t.Helper()
|
||||
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
|
||||
p.pass(ctx)
|
||||
got, err := delivery.DecodePut(puts.values[pkey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
|
||||
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
select {
|
||||
case call := <-bus.calls:
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
composes(t, p.w, "novox", digest)
|
||||
gkey, _ := delivery.PartGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
world(delivery.PartGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||
}
|
||||
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||
t.Fatal("the bus's want was written before the operator's warrant")
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, running)
|
||||
if got := step(); got.Step != delivery.PutHolding {
|
||||
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutReleased {
|
||||
t.Fatalf("released, the put stands at %s", got.Step)
|
||||
}
|
||||
listed, _ := s.doPuts(map[string]any{"module": "nats"})
|
||||
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
|
||||
t.Errorf("puts lists %+v", l)
|
||||
}
|
||||
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
|
||||
t.Errorf("the steps said as events: %v", bus.events)
|
||||
}
|
||||
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
|
||||
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
s.now = func() time.Time { return now.Add(time.Minute) }
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := step()
|
||||
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes2)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
|
||||
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
|
||||
wantsKV.values["want.novox.nats"] = oldWant
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pass(ctx)
|
||||
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
|
||||
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
@@ -122,43 +404,164 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
|
||||
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: delivery.RuleGrant, Words: "no credential"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
|
||||
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
|
||||
for i, c := range []struct {
|
||||
before map[string]delivery.Want
|
||||
out assign.Outcome
|
||||
line string // the line said in this pass, "" for none
|
||||
}{
|
||||
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
|
||||
{nil, notDone, ""},
|
||||
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
if got := strings.Join(logged, "\n"); got != c.line {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
|
||||
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
|
||||
for i, c := range []struct {
|
||||
failed map[string]assign.NodeFailure
|
||||
state string // the condition said in this pass, "" for none
|
||||
}{
|
||||
{failing, asks.StateRaised},
|
||||
{failing, ""},
|
||||
{nil, asks.StateCleared},
|
||||
{nil, ""},
|
||||
} {
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
|
||||
switch {
|
||||
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
|
||||
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
|
||||
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
|
||||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
|
||||
t.Errorf("node failed, raised %+v", conds[0])
|
||||
}
|
||||
}
|
||||
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
|
||||
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
|
||||
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
|
||||
}
|
||||
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
|
||||
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
known assign.Nodes
|
||||
keys []string
|
||||
}{
|
||||
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
} {
|
||||
conds = nil
|
||||
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
|
||||
fresh.logf = func(string, ...any) {}
|
||||
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
|
||||
var keys []string
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
keys = append(keys, cond.Key)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
|
||||
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -189,14 +592,14 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
placeOn(t, p.w, "shanks", "systemd-resolved")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
@@ -236,3 +639,368 @@ func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
|
||||
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
|
||||
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
|
||||
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
|
||||
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
|
||||
trunkOf := func(fp string) map[string]delivery.Module {
|
||||
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
|
||||
}
|
||||
type ask struct{ name, key string }
|
||||
holdOld, _ := delivery.HoldAskKey("web", fpOld)
|
||||
holdNew, _ := delivery.HoldAskKey("web", fpNew)
|
||||
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
|
||||
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
|
||||
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
|
||||
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
|
||||
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
|
||||
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
|
||||
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the node's own composition is not read
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
|
||||
{"the trunk is not read at all", nil, nil, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
for _, key := range all {
|
||||
k, _ := delivery.ParseAnyAskKey(key)
|
||||
var a delivery.AssignerAsk
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold:
|
||||
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
|
||||
case delivery.AskKindRestore:
|
||||
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
|
||||
case delivery.AskKindRecord:
|
||||
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
|
||||
default:
|
||||
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
|
||||
}
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
|
||||
if !c.noDigit {
|
||||
composes(t, p.w, "novox", digNew)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
|
||||
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
p.staleAsks(context.Background(), c.trunk)
|
||||
// the held record and condition of a fingerprint whose hold ask went are taken away with it
|
||||
heldOld, _ = delivery.HeldKey("web", fpOld)
|
||||
_, heldThere := wantsKV.values[heldOld]
|
||||
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
|
||||
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
|
||||
})
|
||||
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
|
||||
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
|
||||
}
|
||||
for _, key := range all {
|
||||
_, there := asksKV.values[key]
|
||||
want := !slices.Contains(c.gone, key)
|
||||
if there != want {
|
||||
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
|
||||
}
|
||||
}
|
||||
if slices.Contains(c.gone, holdOld) {
|
||||
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
|
||||
p.now = func() time.Time { return time.Now().Add(time.Hour) }
|
||||
p.holdAsk(context.Background(), "web", fpOld, nil)
|
||||
p.heldCondition(context.Background(), "web", fpOld, nil, false)
|
||||
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
|
||||
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
|
||||
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
fail := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if fail {
|
||||
fail = false
|
||||
return errors.New("router unreachable")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(holdOld, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[holdOld] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
|
||||
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
|
||||
}
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
|
||||
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
|
||||
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
|
||||
asksKV.values[key] = []byte("x")
|
||||
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
|
||||
}
|
||||
p.lift(context.Background(), "web", fpOld)
|
||||
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
|
||||
if _, there := asksKV.values[key]; there == gone {
|
||||
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func mustKey(key string, err error) string {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
aReleasesApproveAnswersTheMoveOfItsBuildAlone(t)
|
||||
}
|
||||
|
||||
// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is
|
||||
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
|
||||
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
|
||||
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
commit := strings.Repeat("1", 40)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
alone bool
|
||||
at string // the fingerprint the have runs
|
||||
mergedAt string // the commit the release merged web at; "" for no merge of the release
|
||||
fpSrc uint64 // the source revision at which web's fingerprint last changed
|
||||
yes, asked bool
|
||||
}{
|
||||
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
|
||||
{"a setting changed too", false, fpOld, commit, 7, false, true},
|
||||
{"a setting changed at the build it runs", true, fpTest, commit, 7, false, true},
|
||||
{"no approved release carries it", true, fpOld, "", 7, false, true},
|
||||
{"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true},
|
||||
// Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing.
|
||||
{"a fingerprint another release moved", true, fpOld, commit, 8, false, true},
|
||||
// Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the
|
||||
// indexer writes fp-src 0, not known, though Src is still the release's: asked of a person.
|
||||
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
|
||||
} {
|
||||
asksKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
|
||||
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
|
||||
if c.mergedAt != "" {
|
||||
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/web", merge)
|
||||
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/other", other)
|
||||
}
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
|
||||
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
|
||||
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
|
||||
p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out)
|
||||
w := out.Wants["g14"]
|
||||
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
|
||||
t.Errorf("%s: the want is %+v", c.name, w)
|
||||
}
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
if _, there := asksKV.values[akey]; there != c.asked {
|
||||
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
|
||||
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
|
||||
if got := p.w.assigned(); len(got) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v", got)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "c", "a")
|
||||
placeOn(t, p.w, "n2", "d")
|
||||
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
|
||||
t.Errorf("the placement alone: assigned %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
|
||||
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
|
||||
// 8 s, and a want put unchanged each pass made the node compose again each time.
|
||||
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
|
||||
asked := asksKV.values[akey]
|
||||
p.wants(context.Background())
|
||||
if !bytes.Equal(asksKV.values[akey], asked) {
|
||||
t.Fatalf("the record ask the node's have waits on was taken away or made again: %q", asksKV.values[akey])
|
||||
}
|
||||
wantPuts, askPuts := wantsKV.puts, asksKV.puts
|
||||
p.wants(context.Background())
|
||||
if wantsKV.puts != wantPuts || asksKV.puts != askPuts {
|
||||
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
|
||||
}
|
||||
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
|
||||
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
|
||||
if wantsKV.puts != wantPuts {
|
||||
t.Fatalf("a want put unchanged reached the bucket")
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
}
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
|
||||
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
|
||||
// cleared once, and said again at a later pass while the findings cannot be read.
|
||||
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
|
||||
var said []string
|
||||
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
|
||||
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
|
||||
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := 1
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
|
||||
}
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
if len(conds) != 2 || conds[0].State != asks.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
|
||||
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
|
||||
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
|
||||
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
|
||||
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
|
||||
// caller waits on it.
|
||||
|
||||
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
|
||||
const PutEvent = "put"
|
||||
|
||||
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
|
||||
// the seat held once, and an event said. process.Conn, or a test's.
|
||||
type caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
|
||||
// takes the snapshot again, without harm.
|
||||
type putActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
snapping map[string]bool
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
}
|
||||
|
||||
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||
func identity(key string, p delivery.Put) string {
|
||||
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// putsByModule is every put the assigner holds, by module then node.
|
||||
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||
out := map[string]map[string]delivery.Put{}
|
||||
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
put, err := delivery.DecodePut(value)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]delivery.Put{}
|
||||
}
|
||||
out[module][node] = put
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// puts takes every put that has not ended one step further.
|
||||
func (p *passer) puts(ctx context.Context) {
|
||||
if p.acts == nil {
|
||||
return
|
||||
}
|
||||
all := p.putsByModule()
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
var manifest catalogue.Manifest
|
||||
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||
manifest = got.manifest
|
||||
}
|
||||
}
|
||||
for node, put := range all[module] {
|
||||
if delivery.PutEnded(put.Step) {
|
||||
continue
|
||||
}
|
||||
key, _ := delivery.PutKey(node, module)
|
||||
id := identity(key, put)
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||
!strings.Contains(next.Put.Words, failed) {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||
}
|
||||
p.actOn(ctx, node, module, id, world, next)
|
||||
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
|
||||
if !written {
|
||||
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||
}
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
if next.Warranted {
|
||||
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalWant(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
// putWorld is what one step of a put reads from the world.
|
||||
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||
for _, m := range assignedHere {
|
||||
if m == module {
|
||||
w.Assigned = true
|
||||
}
|
||||
}
|
||||
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||
w.Build, w.BuildWords = b.State, b.Error
|
||||
} else {
|
||||
w.Build = "unreadable: " + err.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
if c, ok := chosen[module]; ok {
|
||||
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
|
||||
} else if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
w.Have = &h
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, granted
|
||||
}
|
||||
|
||||
// actOn does the acts a step asks for.
|
||||
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
if next.Snapshot {
|
||||
p.acts.snap(ctx, id, node, module)
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
}
|
||||
|
||||
// samePut says two puts say the same: their step, words and what they found.
|
||||
func samePut(a, b delivery.Put) bool {
|
||||
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||
}
|
||||
|
||||
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||
// released it since; and says it as the event put. It says whether it wrote.
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||
if kv == nil {
|
||||
return false
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
now, err := delivery.DecodePut(e.Value())
|
||||
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||
return false // asked again or released meanwhile: the next pass reads that
|
||||
}
|
||||
data, err := delivery.EncodePut(next)
|
||||
if err != nil {
|
||||
p.logf("the put %s is not written: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
p.acts.say(key, next)
|
||||
return true
|
||||
}
|
||||
|
||||
// state is what this process knows of a put's acts.
|
||||
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.snapped[id], a.snapping[id], a.snapNo[id]
|
||||
}
|
||||
|
||||
// snap has the bus's snapshot taken once for a put, in the background.
|
||||
func (a *putActs) snap(ctx context.Context, id, node, module string) {
|
||||
a.mu.Lock()
|
||||
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.snapping[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
where, err := a.snapshot(ctx, node, module)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.snapping[id] = false
|
||||
if err != nil {
|
||||
a.snapNo[id] = err.Error()
|
||||
return
|
||||
}
|
||||
a.snapped[id] = where
|
||||
}()
|
||||
}
|
||||
|
||||
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
|
||||
var SnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
|
||||
var snapshotLook = 10 * time.Second
|
||||
|
||||
// backedUp is one module as the backup holder's `backed-up` answers it.
|
||||
type backedUp struct {
|
||||
Module string `json:"module"`
|
||||
Data []struct {
|
||||
Item string `json:"item"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
|
||||
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
|
||||
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
|
||||
asked := time.Now()
|
||||
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
|
||||
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
deadline := asked.Add(SnapshotWithin)
|
||||
for {
|
||||
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
|
||||
var modules []backedUp
|
||||
if json.Unmarshal(raw, &modules) == nil {
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
|
||||
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(snapshotLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and
|
||||
// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
|
||||
func (a *putActs) warrantFailed(id string) string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.warrantNo[id]
|
||||
}
|
||||
|
||||
// putSaid is the body of the event put.
|
||||
type putSaid struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Hold bool `json:"hold"`
|
||||
Step string `json:"step"`
|
||||
Words string `json:"words"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
}
|
||||
|
||||
// say says a put's step as the event put, once per step and words.
|
||||
func (a *putActs) say(key string, p delivery.Put) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || a.bus == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.said[key] == p.Step+p.Words {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.said[key] = p.Step + p.Words
|
||||
a.mu.Unlock()
|
||||
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
|
||||
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// shortHex is a digest's first twelve hex digits.
|
||||
func shortHex(digest string) string {
|
||||
h := strings.TrimPrefix(digest, "sha256:")
|
||||
if len(h) > 12 {
|
||||
return h[:12]
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,748 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
|
||||
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
|
||||
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
|
||||
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
|
||||
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
|
||||
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
|
||||
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
|
||||
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
|
||||
|
||||
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
|
||||
type publisher func(event string, body any, id string) error
|
||||
|
||||
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
|
||||
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", delivery.AssignerModule)
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
|
||||
// the controller: placement is the one source (ADR 0567 §8, issue 774).
|
||||
type seatActs struct {
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
|
||||
}
|
||||
|
||||
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
|
||||
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
}
|
||||
ask, _, _ := strings.Cut(warrant, "@")
|
||||
go func() {
|
||||
if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
} else {
|
||||
a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
|
||||
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
|
||||
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
|
||||
func (p *passer) clearStaleCalls() {
|
||||
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
|
||||
return
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.callsCleared = true
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "module.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
|
||||
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
|
||||
if node, module, two := strings.Cut(pair, "."); two {
|
||||
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
|
||||
p.callsCleared = false
|
||||
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
func (p *passer) sayWaits(unbound []string) {
|
||||
c := assign.SeatWaitsCondition(unbound)
|
||||
state := string(c.State) + c.Summary
|
||||
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
|
||||
return
|
||||
}
|
||||
if c.State == asks.StateCleared && p.saidWaits == "" {
|
||||
return // nothing was raised by this process
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key], p.saidWaits = true, state
|
||||
}
|
||||
|
||||
// seatPlacements is every node's placement and the revision of its node key (its legacy key's for a node not yet
|
||||
// converted), as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
|
||||
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
|
||||
return p.w.placement.AllRevisions()
|
||||
}
|
||||
|
||||
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
|
||||
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
|
||||
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
|
||||
var newest time.Time
|
||||
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
out[node], revs[node] = c, meta.revision
|
||||
if meta.at.After(newest) {
|
||||
newest = meta.at
|
||||
}
|
||||
}
|
||||
return out, revs, newest
|
||||
}
|
||||
|
||||
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
|
||||
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[id]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placementSeat takes every open proposal one step further.
|
||||
func (p *passer) placementSeat(ctx context.Context) {
|
||||
for _, b := range seatBuckets {
|
||||
if p.kvs[b] == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.saidProposed == nil {
|
||||
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
current, revs := p.seatPlacements()
|
||||
ids := make([]string, 0)
|
||||
raws := p.w.read(delivery.PlacementProposalsBucket)
|
||||
for id := range raws {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
prop, err := delivery.DecodePlacementProposal(raws[id])
|
||||
if err != nil || prop.ID != id {
|
||||
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
|
||||
continue
|
||||
}
|
||||
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
|
||||
}
|
||||
}
|
||||
|
||||
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
|
||||
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
|
||||
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
|
||||
kv := p.kvs[delivery.PlacementPulsesBucket]
|
||||
_ = kv.Delete(ctx, key)
|
||||
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
|
||||
if err != nil {
|
||||
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
|
||||
}
|
||||
|
||||
// pulsing says whether a bound's key stands.
|
||||
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
|
||||
|
||||
// deletePulses deletes every bound of a proposal.
|
||||
func (p *passer) deletePulses(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementPulsesBucket) {
|
||||
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
|
||||
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
|
||||
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
|
||||
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
|
||||
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
|
||||
return prop, true
|
||||
}
|
||||
|
||||
// stepProposal takes one proposal one step further.
|
||||
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
|
||||
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
|
||||
p.saidProposed[prop.ID] = true
|
||||
}
|
||||
if prop.Withdrawn != "" {
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
if prop.Seed || prop.HandBack {
|
||||
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
|
||||
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
|
||||
return
|
||||
}
|
||||
if prop.Warrant != nil {
|
||||
p.addNode(ctx, prop, revs)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
return
|
||||
case !p.pulsing(askPulse) && !again:
|
||||
p.endProposal(ctx, prop, "its ask expired unanswered")
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
}
|
||||
cands, candRevs, written := p.candidatesOf(prop.ID)
|
||||
if len(prop.Must) == 0 {
|
||||
p.mustAnswer(ctx, prop, rev, current, cands)
|
||||
return
|
||||
}
|
||||
for _, node := range prop.Must {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // a candidate just written is not heard yet: its watch wakes the next pass
|
||||
}
|
||||
}
|
||||
if written.After(prop.Written) {
|
||||
prop.Written = written
|
||||
}
|
||||
pulsing := map[string]bool{}
|
||||
for _, node := range prop.Must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
pulsing[node] = p.pulsing(k)
|
||||
}
|
||||
}
|
||||
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
|
||||
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
|
||||
return
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case delivery.VerdictFail:
|
||||
prop.State, prop.Ask = delivery.ProposalFailed, ""
|
||||
case delivery.VerdictWaits:
|
||||
prop.State, prop.Ask = delivery.ProposalAnswering, ""
|
||||
case delivery.VerdictPass:
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
}
|
||||
if again {
|
||||
prop.Words = "judged and asked again (again)"
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.writeProposal(ctx, prop, rev) {
|
||||
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
|
||||
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
|
||||
}
|
||||
}
|
||||
|
||||
// sameVerdict says two verdicts say the same, their time aside.
|
||||
func sameVerdict(a, b delivery.PlacementVerdict) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
|
||||
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
|
||||
for node, base := range prop.Base {
|
||||
if revs[node] != base {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
|
||||
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
|
||||
key := fmt.Sprint(prop.ID, prop.Base)
|
||||
if got, ok := p.intended[key]; ok {
|
||||
return got
|
||||
}
|
||||
got, _ := p.compute(prop, current, cands)
|
||||
p.intended[key] = got
|
||||
return got
|
||||
}
|
||||
|
||||
// compute is what the proposal changes on each node by its placement, and the nodes that answer it: every node with a
|
||||
// placement or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
|
||||
// change in its answer.
|
||||
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string) {
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
if c, ok := cands[node]; ok && !c.Reached {
|
||||
laid[node] = c.Placement
|
||||
}
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for n := range current {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range p.w.records() {
|
||||
nodes[n] = true
|
||||
}
|
||||
return judge.Intended(current, laid, slices.Sorted(maps.Keys(nodes)))
|
||||
}
|
||||
|
||||
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
|
||||
// change, and writes them into the proposal.
|
||||
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
|
||||
for _, node := range prop.Nodes() {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // the verbs write the candidates before the proposal: wait to hear them
|
||||
}
|
||||
}
|
||||
intended, must := p.compute(prop, current, cands)
|
||||
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
|
||||
for _, node := range must {
|
||||
if _, ok := cands[node]; ok {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
|
||||
if pl, ok := current[node]; ok {
|
||||
c.Placement = &pl
|
||||
}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
for _, node := range must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
p.pulse(ctx, k, judge.AnswerBound)
|
||||
}
|
||||
}
|
||||
prop.Must = must
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
|
||||
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
|
||||
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
for _, node := range again.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
|
||||
// nodes with a record.
|
||||
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
|
||||
held := map[string]bool{}
|
||||
for key := range p.w.read(delivery.ModulesBucket) {
|
||||
if name, err := delivery.ParseModulesKey(key); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
records := p.w.records()
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, and placement-changed said.
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
}
|
||||
cands, _, _ := p.candidatesOf(prop.ID)
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
laid[node] = cands[node].Placement
|
||||
}
|
||||
writes, err := assign.Writes(prop, laid)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
var changes []delivery.PlacementChange
|
||||
writtenLaid := map[string]*delivery.Placement{}
|
||||
after := maps.Clone(current)
|
||||
afterRevs := maps.Clone(revs)
|
||||
for _, w := range writes {
|
||||
rev2, err := p.writeNode(ctx, w.Node, w.Value)
|
||||
if err != nil {
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
before := current[w.Node]
|
||||
now := delivery.Placement{Node: w.Node}
|
||||
if w.Value != nil {
|
||||
now = *w.Value
|
||||
after[w.Node] = *w.Value
|
||||
} else {
|
||||
delete(after, w.Node)
|
||||
}
|
||||
afterRevs[w.Node] = rev2
|
||||
c := delivery.ChangeOf(before, now)
|
||||
c.Before, c.After = w.Base, rev2
|
||||
changes = append(changes, c)
|
||||
writtenLaid[w.Node] = w.Value
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes)
|
||||
}
|
||||
|
||||
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
|
||||
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
|
||||
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
|
||||
// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
|
||||
func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) {
|
||||
current, _ := p.seatPlacements()
|
||||
if moved(prop, revs) {
|
||||
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
|
||||
return
|
||||
}
|
||||
laid, findings := delivery.LayChanges(current, prop.Changes)
|
||||
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
|
||||
node := prop.Changes[0].Node
|
||||
if prop.Changes[0].Op == delivery.OpRemoveNode {
|
||||
p.removeNode(ctx, prop, current, findings)
|
||||
return
|
||||
}
|
||||
value, ok := laid[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, &value)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(delivery.Placement{Node: node}, value)
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
|
||||
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant.
|
||||
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
|
||||
func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) {
|
||||
node := prop.Changes[0].Node
|
||||
before, ok := current[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, nil)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
|
||||
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
|
||||
at = t
|
||||
}
|
||||
}
|
||||
delivery.SortChanges(changes)
|
||||
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
|
||||
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
|
||||
p.w.put(delivery.AssignerAsksBucket, akey, nil)
|
||||
}
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
// deleteCandidates deletes every candidate of a proposal.
|
||||
func (p *passer) deleteCandidates(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
|
||||
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
|
||||
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
|
||||
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
|
||||
raw, err := delivery.EncodePlacementProposal(prop)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s is not written: %v", prop.ID, err)
|
||||
return false
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementProposalsBucket]
|
||||
var at uint64
|
||||
if rev == 0 {
|
||||
at, err = kv.Create(ctx, prop.ID, raw)
|
||||
} else {
|
||||
at, err = kv.Update(ctx, prop.ID, raw, rev)
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
|
||||
return true
|
||||
}
|
||||
|
||||
// say says one of the seat's events, said in the journal when the bus refuses it.
|
||||
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
|
||||
if p.publish == nil {
|
||||
return
|
||||
}
|
||||
if err := p.publish(event, body, id); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// writeNode writes one node's placement after (nil: none) as its keys (mesh-sdk delivery.PlacementKeys.Writes; research
|
||||
// 072 row 37): the module keys that change, each carrying the value it replaces, then the node key, the commit, then the
|
||||
// module keys no longer placed. It answers the node key's revision after (0: deleted). A write that fails up to the
|
||||
// commit ends it with nothing changed for any reader, the module keys written before it read at their previous values;
|
||||
// a delete after the commit that fails leaves a module key the node key does not name, which places nothing and is
|
||||
// deleted at the node's next write. Each key written is taken into the world at once, so the next write of the pass
|
||||
// reads it.
|
||||
func (p *passer) writeNode(ctx context.Context, node string, after *delivery.Placement) (uint64, error) {
|
||||
writes, err := p.w.placement.Writes(node, after)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementBucket]
|
||||
committed, rev := false, uint64(0)
|
||||
for _, w := range writes {
|
||||
got, err := writeKey(ctx, kv, w)
|
||||
if err != nil {
|
||||
if committed {
|
||||
p.logf("placement %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
|
||||
continue
|
||||
}
|
||||
return 0, fmt.Errorf("placement %s at revision %d: %w", w.Key, w.Base, err)
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementBucket, w.Key, w.Value, got, p.clock())
|
||||
if w.Commit {
|
||||
committed, rev = true, got
|
||||
}
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// writeKey makes one write of a placement key by compare-and-set and answers the key's revision after (0: deleted).
|
||||
func writeKey(ctx context.Context, kv jetstream.KeyValue, w delivery.PlacementKeyWrite) (uint64, error) {
|
||||
switch {
|
||||
case w.Value == nil:
|
||||
return 0, kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
|
||||
case w.Base == 0:
|
||||
return kv.Create(ctx, w.Key, w.Value)
|
||||
default:
|
||||
return kv.Update(ctx, w.Key, w.Value, w.Base)
|
||||
}
|
||||
}
|
||||
|
||||
// convertPlacement converts every node the state holds only in the form before, `placement.<node>`, into its node key
|
||||
// and module keys (research 072 row 37), once: each node converted is said, and a node whose conversion fails is said
|
||||
// and tried again at the next pass. The legacy key is left for the readers not yet delivered; a node key wins over it
|
||||
// for every reader that reads both, and it is deleted with the node key when the node's placement is.
|
||||
func (p *passer) convertPlacement(ctx context.Context) {
|
||||
if p.convertedAll {
|
||||
return
|
||||
}
|
||||
migrations, err := p.w.placement.Migrations()
|
||||
if err != nil {
|
||||
p.logf("placement: some nodes in the form before cannot be converted, and stay read from it: %v", err)
|
||||
}
|
||||
failed := false
|
||||
for _, node := range slices.Sorted(maps.Keys(migrations)) {
|
||||
before, rev, _ := p.w.placement.Of(node)
|
||||
got, err := p.writeNode(ctx, node, &before)
|
||||
if err != nil {
|
||||
failed = true
|
||||
p.logf("placement: %s is not converted from placement.%s (revision %d) to its node and module keys, and is tried again at the next pass: %v", node, node, rev, err)
|
||||
continue
|
||||
}
|
||||
p.logf("placement: %s converted from placement.%s (revision %d) to %s.node (revision %d) and %d module keys (research 072 row 37); placement.%s stays for readers not yet delivered",
|
||||
node, node, rev, node, got, len(before.Modules), node)
|
||||
}
|
||||
p.convertedAll = !failed && err == nil
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
|
||||
|
||||
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
|
||||
type seatStore interface {
|
||||
store
|
||||
del(state, key string) error
|
||||
history(state, key string) ([]stdio.StateRevision, error)
|
||||
}
|
||||
|
||||
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
held := delivery.NewPlacementKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
out, revs := held.AllRevisions()
|
||||
if len(bad) > 0 && len(out) == 0 {
|
||||
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
raw = []byte(c)
|
||||
case nil:
|
||||
return nil, errors.New("propose names its changes")
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// doPropose is the verb propose (§2).
|
||||
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
changes, err := changesOf(args["changes"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warrant, err := warrantOf(args["warrant"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
|
||||
if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
|
||||
func warrantOf(v any) (*delivery.AddNodeWarrant, error) {
|
||||
var raw []byte
|
||||
switch w := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
if strings.TrimSpace(w) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw = []byte(w)
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out delivery.AddNodeWarrant
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
|
||||
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
|
||||
// as proven, which the refusal names.
|
||||
func moduleOfCaller(by string) string {
|
||||
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
|
||||
return delivery.EnrolmentModule
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
|
||||
}
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
}
|
||||
|
||||
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
|
||||
// when its verdict passes.
|
||||
func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
History int `json:"history-per-key"`
|
||||
Changes []historyEntry `json:"changes,omitempty"`
|
||||
Older string `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
if arg(args, "seats") == "true" {
|
||||
return s.seats(arg(args, "node"))
|
||||
}
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
}
|
||||
nodes := out.Nodes
|
||||
if node != "" {
|
||||
nodes = []string{node}
|
||||
} else if module == "" {
|
||||
nodes = nil
|
||||
for n := range all {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
revisions, err := s.nodeHistory(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var before sketch
|
||||
for _, r := range revisions {
|
||||
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
|
||||
after := sketch{}
|
||||
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
|
||||
after = sketchOf(r.Key, r.Value)
|
||||
} else {
|
||||
e.Deleted = true
|
||||
}
|
||||
e.Change = after.changeFrom(n, before)
|
||||
e.Change.Before, e.Change.After = 0, r.Revision
|
||||
before = after
|
||||
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := []openProposal{}
|
||||
for _, id := range ids {
|
||||
if want := arg(args, "id"); want != "" && want != id {
|
||||
continue
|
||||
}
|
||||
p, err := s.proposal(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if arg(args, "id") != "" && len(out) == 0 {
|
||||
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
|
||||
}
|
||||
return map[string]any{"proposals": out}, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
|
||||
// only from a tool of that name), with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
keys, err := s.st.keys(delivery.SeatHoldersState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []delivery.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var sh delivery.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
|
||||
continue
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return map[string]any{"seats": out, "holds": len(keys)}, nil
|
||||
}
|
||||
|
||||
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
|
||||
type keyedRevision struct {
|
||||
stdio.StateRevision
|
||||
Key string
|
||||
}
|
||||
|
||||
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
|
||||
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
|
||||
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
|
||||
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
|
||||
legacy, err := delivery.LegacyPlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodeKey, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []keyedRevision
|
||||
for _, key := range []string{legacy, nodeKey} {
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, r := range revisions {
|
||||
out = append(out, keyedRevision{StateRevision: r, Key: key})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
|
||||
// and holdings, which change every module (delivery.ChangeOf).
|
||||
type sketch struct {
|
||||
modules []delivery.PlacementRef
|
||||
nodeWide string
|
||||
}
|
||||
|
||||
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
|
||||
func sketchOf(key string, value []byte) sketch {
|
||||
_, _, kind, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
|
||||
raw, _ := json.Marshal([]any{pins, holds})
|
||||
return string(raw)
|
||||
}
|
||||
if kind == delivery.PlacementNodeKind {
|
||||
n, err := delivery.DecodePlacementNode(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
|
||||
for _, m := range p.Modules {
|
||||
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
|
||||
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
|
||||
c := delivery.PlacementChange{Node: node}
|
||||
was := map[string]string{}
|
||||
for _, r := range before.modules {
|
||||
was[r.Module] = r.Digest
|
||||
}
|
||||
now := map[string]bool{}
|
||||
nodeWide := before.nodeWide != s.nodeWide && len(before.modules) > 0
|
||||
for _, r := range s.modules {
|
||||
now[r.Module] = true
|
||||
d, ok := was[r.Module]
|
||||
switch {
|
||||
case !ok:
|
||||
c.Added = append(c.Added, r.Module)
|
||||
case nodeWide || d != r.Digest:
|
||||
c.Changed = append(c.Changed, r.Module)
|
||||
}
|
||||
}
|
||||
for _, r := range before.modules {
|
||||
if !now[r.Module] {
|
||||
c.Removed = append(c.Removed, r.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(c.Added)
|
||||
sort.Strings(c.Removed)
|
||||
sort.Strings(c.Changed)
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
func (s memStore) del(state, key string) error {
|
||||
delete(s[state].values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
v, ok := s[state].values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
st := memStore{}
|
||||
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
for _, m := range []string{"web", "shell"} {
|
||||
st[modulesState].values[m] = []byte("{}")
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
|
||||
t.Errorf("marked = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
|
||||
t.Error("an unknown proposal was withdrawn")
|
||||
}
|
||||
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
t.Error("an unknown change was taken")
|
||||
}
|
||||
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
|
||||
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
move := got.(proposed).Proposal
|
||||
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
|
||||
t.Fatalf("propose = %+v", move)
|
||||
}
|
||||
|
||||
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
|
||||
reads := map[string]bool{}
|
||||
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
|
||||
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
|
||||
t.Errorf("placement of web = %+v, %v", pa, err)
|
||||
} else {
|
||||
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
|
||||
}
|
||||
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
|
||||
t.Error("a node with no key was answered")
|
||||
}
|
||||
ps, err := s.doProposals(map[string]any{"id": move.ID})
|
||||
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
|
||||
t.Errorf("proposals = %+v, %v", ps, err)
|
||||
} else {
|
||||
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
|
||||
}
|
||||
raw0, err := readManifest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range raw0.State {
|
||||
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
|
||||
t.Errorf("no verb reads the seat's state %s", state.Name)
|
||||
}
|
||||
}
|
||||
for _, stream := range raw0.Streams {
|
||||
if !reads["stream "+stream.Name] {
|
||||
t.Errorf("no verb reads the seat's stream %s", stream.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold: %v", err)
|
||||
}
|
||||
// 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with
|
||||
// the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused.
|
||||
addNode := func(by, warrant string) error {
|
||||
_, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace",
|
||||
"decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by)
|
||||
return err
|
||||
}
|
||||
good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`
|
||||
for _, c := range []struct{ by, warrant, want string }{
|
||||
{"novox.mesh-grants", good, "mesh-enrolment's alone"},
|
||||
{"person.mesh-enrolment", good, "mesh-enrolment's alone"},
|
||||
{"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
{"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"},
|
||||
{"novox.mesh-enrolment", "", "carries no warrant"},
|
||||
} {
|
||||
if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want)
|
||||
}
|
||||
}
|
||||
if err := addNode("novox.mesh-enrolment", good); err != nil {
|
||||
t.Errorf("an add-node from mesh-enrolment: %v", err)
|
||||
}
|
||||
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
|
||||
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
|
||||
// of a node running modules, or with another node's warrant, it is refused.
|
||||
joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}})
|
||||
st[delivery.PlacementState].values["placement.joiner"] = joiner
|
||||
removeNode := func(node, warrant string) (any, error) {
|
||||
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
|
||||
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
|
||||
}
|
||||
for _, c := range []struct{ node, warrant, want string }{
|
||||
{"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"},
|
||||
{"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
} {
|
||||
if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want)
|
||||
}
|
||||
}
|
||||
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
|
||||
t.Errorf("a remove-node from mesh-enrolment: %v", err)
|
||||
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
|
||||
t.Errorf("the remove-node proposed %+v", rm)
|
||||
}
|
||||
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
|
||||
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey, _ := delivery.LegacyPlacementKey("ace")
|
||||
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
|
||||
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
|
||||
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
hk, _ := delivery.HaveNodeKey(node)
|
||||
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
|
||||
}
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Errorf("ace: %+v %v", d, ok)
|
||||
}
|
||||
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
|
||||
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
}
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
p.unplacedFindings()
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey2, _ := delivery.LegacyPlacementKey("g14")
|
||||
w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now())
|
||||
p.unplacedFindings()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("placed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
|
||||
writes := 0
|
||||
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
|
||||
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}}
|
||||
place := func(node string, holds ...delivery.Holding) {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := delivery.LegacyPlacementKey(node)
|
||||
p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now())
|
||||
}
|
||||
place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
ctx := context.Background()
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
|
||||
t.Fatalf("%d writes: %v", writes, kv.values)
|
||||
}
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 {
|
||||
t.Errorf("an unchanged placement wrote %d keys", writes-3)
|
||||
}
|
||||
place("g14") // g14 holds nothing now
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
|
||||
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
|
||||
}
|
||||
st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}}
|
||||
out, err := newSeatTools(st, nil).seats("ace")
|
||||
if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 {
|
||||
t.Errorf("seats on ace = %+v, %v", out, err)
|
||||
}
|
||||
})
|
||||
t.Run("research 072 row 37: the form before is converted once, a node is written as its keys, and history reads both", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var logs []string
|
||||
p := &passer{w: newWorld(), kvs: buckets{delivery.PlacementBucket: kv}, now: time.Now,
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
put := func(key string, value []byte, rev uint64) {
|
||||
p.w.putEntry(delivery.PlacementBucket, key, value, rev, time.Now())
|
||||
}
|
||||
was := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
raw, _ := delivery.EncodePlacement(was)
|
||||
put("placement.anchor", raw, kv.write("placement.anchor", raw))
|
||||
p.convertPlacement(ctx)
|
||||
if strings.Join(kv.log, " ") != "create anchor.web create anchor.node" || !p.convertedAll ||
|
||||
!strings.Contains(strings.Join(logs, "\n"), "anchor converted from placement.anchor") {
|
||||
t.Fatalf("the conversion wrote %v, said %v", kv.log, logs)
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
if len(kv.log) != 2 {
|
||||
t.Errorf("a converted node is converted again: %v", kv.log)
|
||||
}
|
||||
// A write: the module keys, the node key (its commit), the mirror, then the module key no longer placed.
|
||||
after := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2222}}}}
|
||||
kv.log = nil
|
||||
rev, err := p.writeNode(ctx, "anchor", &after)
|
||||
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node update placement.anchor delete anchor.web" || rev != kv.revs["anchor.node"] {
|
||||
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
|
||||
}
|
||||
if got, _ := p.seatPlacements(); !delivery.SamePlacement(got["anchor"], after) {
|
||||
t.Errorf("the write is not read back at once: %+v", got)
|
||||
}
|
||||
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
|
||||
kv.refuse, kv.log = "anchor.node", nil
|
||||
moved := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2223}}}}
|
||||
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
|
||||
t.Fatalf("a refused commit = %v, %v", kv.log, err)
|
||||
}
|
||||
fresh, err := delivery.PlacementsOf(kv.values)
|
||||
if err != nil || !delivery.SamePlacement(fresh["anchor"], after) {
|
||||
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
|
||||
}
|
||||
kv.refuse = ""
|
||||
// The verb's history reads the form before, then the node key.
|
||||
st := memStore{delivery.PlacementState: &memKV{values: kv.values}}
|
||||
hs := newSeatTools(st, nil)
|
||||
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
|
||||
"placement.anchor": {{Key: "placement.anchor", Value: raw, Revision: 1, Op: "put"}},
|
||||
"anchor.node": {{Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
|
||||
}}
|
||||
pa, err := hs.doPlacement(map[string]any{"node": "anchor", "history": "true"})
|
||||
a, _ := pa.(placementAnswer)
|
||||
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
|
||||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
|
||||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
|
||||
t.Errorf("history = %+v, %v", pa, err)
|
||||
}
|
||||
// No placement: the mirror first, then the node key, then the module keys.
|
||||
kv.log = nil
|
||||
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete placement.anchor delete anchor.node delete anchor.shell" {
|
||||
t.Errorf("the delete = %v, %v", kv.log, err)
|
||||
}
|
||||
})
|
||||
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now())
|
||||
p.kvs[provisions.ProvisionsBucket] = &memKV{}
|
||||
of := p.provisionsOf()
|
||||
if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed {
|
||||
t.Fatalf("provisions = %+v", got)
|
||||
}
|
||||
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now())
|
||||
p.provisionsOf()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings after provided = %v", findings)
|
||||
}
|
||||
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
|
||||
ctx := context.Background()
|
||||
p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
|
||||
gone := map[string]bool{"ace.photos.s3": true}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
|
||||
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if len(findings) != 0 {
|
||||
t.Fatal("said before its bound")
|
||||
}
|
||||
p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
|
||||
t.Fatalf("past its bound: findings %v", findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, map[string]bool{})
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("keyed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
|
||||
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
|
||||
func wantsVerbs(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||
st := memStore{}
|
||||
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
|
||||
// a fingerprint, and laptop's web refused there.
|
||||
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
|
||||
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
|
||||
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
|
||||
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
|
||||
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
|
||||
manifest := func(name, extra string) {
|
||||
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
|
||||
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
|
||||
}
|
||||
manifest("web", "")
|
||||
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
|
||||
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
var asked []string
|
||||
var said []string
|
||||
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
}, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
|
||||
policyOf := func(module string) policyRow {
|
||||
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(map[string]any)["modules"].([]policyRow)[0]
|
||||
}
|
||||
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
|
||||
t.Errorf("photos = %+v", p)
|
||||
}
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { // the policy its manifest says (issue 774)
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
|
||||
// controller being gone (1126/B8).
|
||||
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
|
||||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
|
||||
t.Errorf("compare = %+v", got)
|
||||
}
|
||||
pub, _ := out.(map[string]any)["published"].(map[string][]publishedRow)
|
||||
if a, l := pub["anchor"], pub["laptop"]; len(a) != 2 || a[0].Module != "node" || a[1].Module != "web" || a[1].FP != fpNew ||
|
||||
!strings.HasPrefix(a[1].Digest, "sha256:") || len(l) != 1 || l[0].Refused != "a required value is unset" || l[0].Digest != "" {
|
||||
t.Errorf("published = %+v (unread %v)", pub, out.(map[string]any)["published-unread"])
|
||||
}
|
||||
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
|
||||
// large states (the modules, their manifests, the composed record).
|
||||
watches := map[string]int{}
|
||||
byWatch := newWantsTools(st, nil, nil)
|
||||
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
watches[state]++
|
||||
for k, v := range st[state].values {
|
||||
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true})
|
||||
}
|
||||
return nil
|
||||
}, func(state, key string) ([]byte, bool, error) {
|
||||
watches[state+" by key"]++
|
||||
v, ok := st[state].values[key]
|
||||
return v, ok, nil
|
||||
})
|
||||
for range 2 {
|
||||
wout, werr := byWatch.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("compare by watch = %+v", got)
|
||||
}
|
||||
}
|
||||
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
|
||||
t.Errorf("the manifests were read by key %d times, want each once", n)
|
||||
}
|
||||
for _, state := range []string{modulesState, haveState, health.FindingsState} {
|
||||
if watches[state] != 1 {
|
||||
t.Errorf("%s was watched %d times, want once", state, watches[state])
|
||||
}
|
||||
}
|
||||
// ADR 0580 clause 3: the compare reads through mesh-sdk's watched reader and the seat keeps no mirror of its own.
|
||||
for f := range reflect.TypeFor[wantsTools]().Fields() {
|
||||
if f.Type.Kind() == reflect.Map || (f.Type.Kind() == reflect.Pointer && f.Type.Elem().Kind() == reflect.Map) {
|
||||
t.Errorf("wantsTools keeps %s, a copy of its own beside the watched reader", f.Name)
|
||||
}
|
||||
}
|
||||
if watches[policiesState] != 0 {
|
||||
t.Errorf("the seat's own small policies state was watched: %v", watches)
|
||||
}
|
||||
// a person's record for web: written, recorded, said.
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chosen, _ := assign.DecodePolicies(st[policiesState].values)
|
||||
if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") ||
|
||||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
|
||||
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
|
||||
}
|
||||
// A person may choose the bus's policy as any module's (ADR 0601 §1: no part treats the module that provides the bus apart).
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err != nil {
|
||||
t.Errorf("a person's policy for the bus was refused: %v", err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "default", "why": "back to the module"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
|
||||
t.Error("a policy without why was set")
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 {
|
||||
t.Errorf("default left %v (%v)", st[policiesState].values, err)
|
||||
}
|
||||
// bus: the node behind the indexer's newest.
|
||||
b, err := w.doBus(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodes := b.(map[string]any)["nodes"].([]busNode)
|
||||
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
|
||||
t.Errorf("bus = %+v, asked %v", b, asked)
|
||||
}
|
||||
if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// readManifest is the module's manifest, read as the controller reads it.
|
||||
func readManifest() (catalogue.Manifest, error) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.Decode(raw)
|
||||
}
|
||||
|
||||
// countingKV is a memKV that counts its puts.
|
||||
type countingKV struct {
|
||||
memKV
|
||||
writes *int
|
||||
}
|
||||
|
||||
func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
*c.writes++
|
||||
return c.memKV.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// revKV is a bucket with real revisions, refusing a write at a revision that is not the key's, and a write of refuse;
|
||||
// log is every write in order.
|
||||
type revKV struct {
|
||||
jetstream.KeyValue
|
||||
rev uint64
|
||||
values map[string][]byte
|
||||
revs map[string]uint64
|
||||
log []string
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (k *revKV) write(key string, value []byte) uint64 {
|
||||
k.rev++
|
||||
k.values[key], k.revs[key] = value, k.rev
|
||||
return k.rev
|
||||
}
|
||||
|
||||
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if _, ok := k.values[key]; ok || key == k.refuse {
|
||||
return 0, errors.New("key exists")
|
||||
}
|
||||
k.log = append(k.log, "create "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if k.revs[key] != revision || key == k.refuse {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "update "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
if key == k.refuse {
|
||||
return errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "delete "+key)
|
||||
delete(k.values, key)
|
||||
delete(k.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// historyStore is a memStore whose history is given.
|
||||
type historyStore struct {
|
||||
memStore
|
||||
revisions map[string][]stdio.StateRevision
|
||||
}
|
||||
|
||||
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
|
||||
return s.revisions[key], nil
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
|
||||
var answerWithin = 20 * time.Second
|
||||
|
||||
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
|
||||
type store interface {
|
||||
get(state, key string) ([]byte, uint64, bool, error)
|
||||
put(state, key string, value []byte) error
|
||||
keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||
type putTools struct {
|
||||
st store
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||
heard map[string][]byte
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newPutTools(st store) *putTools {
|
||||
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// hear takes a put's change in from the watch.
|
||||
func (s *putTools) hear(key string, value []byte) {
|
||||
s.mu.Lock()
|
||||
s.heard[key] = value
|
||||
close(s.changed)
|
||||
s.changed = make(chan struct{})
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||
var (
|
||||
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||
)
|
||||
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||
// of one the builder built, which must name one build alone.
|
||||
func (s *putTools) version(module, version string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(version))
|
||||
switch {
|
||||
case v == "newest":
|
||||
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if m.FP2 == "" {
|
||||
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||
}
|
||||
return m.FP2, nil
|
||||
case fullFP.MatchString(v):
|
||||
return v, nil
|
||||
case prefix.MatchString(v):
|
||||
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||
found = append(found, fp)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
sort.Strings(found)
|
||||
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||
}
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||
}
|
||||
|
||||
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||
type putAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
delivery.Put
|
||||
Follow string `json:"follow,omitempty"`
|
||||
}
|
||||
|
||||
// doPut is the tool assigner_put.
|
||||
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("put says why: it is kept with the put and said in any ask")
|
||||
}
|
||||
hold := false
|
||||
switch arg(args, "hold") {
|
||||
case "true":
|
||||
hold = true
|
||||
case "", "false":
|
||||
default:
|
||||
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
|
||||
}
|
||||
fp, err := s.version(module, arg(args, "version"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now().UTC()
|
||||
words := fmt.Sprintf("asked by %s: %s", by, why)
|
||||
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
} else if found {
|
||||
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||
if before.FP == fp && before.Hold == hold {
|
||||
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||
}
|
||||
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||
}
|
||||
}
|
||||
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||
}
|
||||
|
||||
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||
deadline := time.After(answerWithin)
|
||||
for {
|
||||
s.mu.Lock()
|
||||
raw, changed := s.heard[key], s.changed
|
||||
s.mu.Unlock()
|
||||
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||
p = now
|
||||
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||
return p // it waits on a person or the node, or ended
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-changed:
|
||||
case <-deadline:
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// followPut says how a caller follows a put after the answer.
|
||||
func followPut(node, module string) string {
|
||||
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
|
||||
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
|
||||
}
|
||||
|
||||
// doRelease is the tool assigner_release.
|
||||
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept with the put")
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||
}
|
||||
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||
}
|
||||
|
||||
// doPuts is the tool assigner_puts.
|
||||
func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(putsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []putAnswer{}
|
||||
for _, key := range keys {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||
}
|
||||
return map[string]any{"puts": out}, nil
|
||||
}
|
||||
|
||||
func arg(args map[string]any, name string) string {
|
||||
switch v := args[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The tools' names, as the manifest lists them.
|
||||
const (
|
||||
toolPut = "assigner_put"
|
||||
toolRelease = "assigner_release"
|
||||
toolPuts = "assigner_puts"
|
||||
)
|
||||
|
||||
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if slices.Contains(switches, name) {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
|
||||
func (s *putTools) tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
|
||||
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
|
||||
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
|
||||
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
|
||||
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine, where the module is assigned",
|
||||
"module": "the module",
|
||||
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
|
||||
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
|
||||
"why": "why, kept with the put and said in any ask",
|
||||
}, []string{"node", "module", "version", "why"}, "hold"),
|
||||
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||
[]string{"node", "module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
Run: s.doPuts},
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeStore is the runtime's states.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) put(state, key string, value []byte) error {
|
||||
_, err := stdio.State(state).Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||
func serveTools() error {
|
||||
s := newPutTools(runtimeStore{})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
|
||||
s.hear(c.Key, c.Value)
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
|
||||
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
|
||||
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
|
||||
return stdio.SeatPublish(wantsEventSubject(e), b, id)
|
||||
})
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
tools = append(tools, wants.tools()...)
|
||||
return stdio.Serve("", tools)
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
func shortFP(fp string) string {
|
||||
part, err := delivery.FingerprintKeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
return fp
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
busStepCause = "bus-upgrade"
|
||||
)
|
||||
|
||||
// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit.
|
||||
type wantsTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
emit func(event, id string, body any) error
|
||||
now func() time.Time
|
||||
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
|
||||
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
|
||||
// a large state (issue 1953 L5f). Nil reads key by key.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
// wantsEventSubject is the subject of one of the seat's events.
|
||||
func wantsEventSubject(event string) string {
|
||||
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
|
||||
}
|
||||
|
||||
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Values(state)
|
||||
}
|
||||
return s.direct(state)
|
||||
}
|
||||
|
||||
// direct reads a state key by key, for a small state of the seat's own.
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
|
||||
// under its digest.
|
||||
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Once(manifestsState, digest)
|
||||
}
|
||||
raw, _, found, err := s.st.get(manifestsState, digest)
|
||||
return raw, found, err
|
||||
}
|
||||
|
||||
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
|
||||
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
|
||||
mods, err := s.values(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
|
||||
for name, raw := range mods {
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
held[name] = m
|
||||
if m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mraw, found, err := s.manifest(m.Manifest)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(mraw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
|
||||
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
|
||||
out[name] = man
|
||||
}
|
||||
}
|
||||
return out, held, nil
|
||||
}
|
||||
|
||||
// policyRow is one module's policy as upgrade answers it.
|
||||
type policyRow struct {
|
||||
Module string `json:"module"`
|
||||
Policy string `json:"policy"`
|
||||
From string `json:"from"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
|
||||
// back with default; with compare each node's composition and the assigner's findings.
|
||||
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
manifests, _, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
}
|
||||
out := map[string]any{"composed": composed}
|
||||
// And what every node published (ADR 0593 §8), read only to compare.
|
||||
if values, err := s.values(delivery.ComposedRead); err != nil {
|
||||
out["published-unread"] = err.Error()
|
||||
} else {
|
||||
out["published"] = published(values)
|
||||
}
|
||||
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
} else {
|
||||
keys := []string{}
|
||||
for k := range found {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out["findings"] = keys
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
module, policy := arg(args, "module"), arg(args, "policy")
|
||||
if policy != "" {
|
||||
return s.setPolicy(module, policy, arg(args, "why"), by, manifests)
|
||||
}
|
||||
var rows []policyRow
|
||||
for name, m := range manifests {
|
||||
if module != "" && name != module {
|
||||
continue
|
||||
}
|
||||
p, from, why, who := assign.PolicyOf(m, chosen)
|
||||
rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who})
|
||||
}
|
||||
if module != "" && len(rows) == 0 {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
|
||||
return map[string]any{"modules": rows, "unreadable": unreadable}, nil
|
||||
}
|
||||
|
||||
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
|
||||
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
|
||||
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("a policy is set for one module: name it")
|
||||
}
|
||||
if _, ok := manifests[module]; !ok {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("a person's policy says why")
|
||||
}
|
||||
if by == "" || by == stdio.Unproven {
|
||||
return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller")
|
||||
}
|
||||
what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy)
|
||||
if policy == "default" {
|
||||
if err := s.st.del(policiesState, module); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err)
|
||||
}
|
||||
} else {
|
||||
p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()}
|
||||
if err := p.Check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _ := json.Marshal(p)
|
||||
if err := s.st.put(policiesState, module, raw); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := handacts.Recorder{Ask: s.ask}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
recorded = "not recorded: " + rerr.Error()
|
||||
}
|
||||
body := map[string]any{"module": module, "policy": policy, "why": why, "by": by}
|
||||
said := ""
|
||||
if s.emit != nil {
|
||||
id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10)
|
||||
if err := s.emit(PolicyChangedEvent, id, body); err != nil {
|
||||
said = "not said: " + err.Error()
|
||||
}
|
||||
}
|
||||
return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil
|
||||
}
|
||||
|
||||
// busNode is one node's bus as bus answers it.
|
||||
type busNode struct {
|
||||
Node string `json:"node"`
|
||||
FP string `json:"fp"`
|
||||
Behind bool `json:"behind"`
|
||||
}
|
||||
|
||||
// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind,
|
||||
// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only.
|
||||
func (s *wantsTools) doBus(map[string]any) (any, error) {
|
||||
manifests, held, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus := ""
|
||||
for name, m := range manifests {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
bus = name
|
||||
}
|
||||
}
|
||||
if bus == "" {
|
||||
return nil, errors.New("the indexer holds no module that provides the bus")
|
||||
}
|
||||
newest := held[bus].FP2
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nodes []busNode
|
||||
for key, raw := range haves {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode || module != bus {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHave(raw)
|
||||
if err != nil || h.Phase == delivery.PhaseRemoved {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
|
||||
}
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
|
||||
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
|
||||
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
|
||||
out["steps-unread"] = err.Error()
|
||||
} else {
|
||||
out["steps"] = steps
|
||||
}
|
||||
puts, err := s.values(putsState)
|
||||
if err != nil {
|
||||
out["puts-unread"] = err.Error()
|
||||
} else {
|
||||
mine := map[string]json.RawMessage{}
|
||||
for k, v := range puts {
|
||||
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
|
||||
mine[k] = v
|
||||
}
|
||||
}
|
||||
out["puts"] = mine
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
|
||||
func (s *wantsTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
|
||||
"bus": s.doBus,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,12 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -15,7 +12,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -24,18 +20,6 @@ import (
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -46,6 +30,22 @@ type world struct {
|
||||
healthy map[string][]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
|
||||
trunkBuilt map[string]delivery.BuildLogEntry
|
||||
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
|
||||
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
placement *delivery.PlacementKeys
|
||||
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
|
||||
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
|
||||
sourceGroups map[uint64]string
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
type entryMeta struct {
|
||||
revision uint64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -56,7 +56,69 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(),
|
||||
sourceGroups: map[uint64]string{}}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
|
||||
w.put(bucket, key, value)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(w.meta, bucket+"|"+key)
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
}
|
||||
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
|
||||
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
|
||||
w.sourceGroups[revision] = s.Group
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
|
||||
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
|
||||
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
|
||||
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
|
||||
// unknown: none, and the move is asked of a person.
|
||||
func (w *world) release(module, fp string) (string, bool) {
|
||||
t, ok := w.trunk()[module]
|
||||
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
|
||||
return "", false
|
||||
}
|
||||
w.mu.Lock()
|
||||
group := w.sourceGroups[t.FPSrc]
|
||||
w.mu.Unlock()
|
||||
key, err := delivery.MergesKey(group)
|
||||
if group == "" || err != nil {
|
||||
return "", false
|
||||
}
|
||||
raw := w.value(delivery.MergesBucket, key)
|
||||
if raw == nil {
|
||||
return "", false
|
||||
}
|
||||
merge, err := delivery.DecodeMerge(raw)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, pull := range merge.Pulls {
|
||||
if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) {
|
||||
return group, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
func (w *world) metaOf(bucket, key string) entryMeta {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.meta[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -66,12 +128,20 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
if w.values[bucket] == nil {
|
||||
w.values[bucket] = map[string][]byte{}
|
||||
}
|
||||
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
|
||||
if value == nil {
|
||||
delete(w.values[bucket], key)
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, nil, 0, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.PlacementBucket {
|
||||
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
|
||||
_, _ = w.placement.Put(key, value, 0, false)
|
||||
}
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -149,10 +219,10 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
m, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
p = parsed{err: err.Error()}
|
||||
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
|
||||
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
|
||||
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
|
||||
p = parsed{module: m.Module, err: err.Error()}
|
||||
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
|
||||
// module's value where it is read (world.mesh).
|
||||
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
|
||||
} else {
|
||||
p = parsed{module: m.Module, manifest: man}
|
||||
}
|
||||
@@ -162,21 +232,42 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
return out
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
for node, p := range w.placements() {
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, all
|
||||
return out, len(w.read(delivery.PlacementBucket)) > 0
|
||||
}
|
||||
|
||||
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
|
||||
func (w *world) placements() map[string]delivery.Placement {
|
||||
out, _ := w.placement.AllRevisions()
|
||||
return out
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
@@ -194,75 +285,35 @@ func (w *world) trunk() map[string]delivery.Module {
|
||||
return out
|
||||
}
|
||||
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned, _ = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
|
||||
// with those placement names (ADR 0593 §8).
|
||||
func (w *world) records() map[string]delivery.NodeRecord {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
m.Digests[name] = mod.Manifest
|
||||
p := w.manifest(ctx, manifests, mod.Manifest)
|
||||
if p.err != "" {
|
||||
m.Unreadable[name] = p.err
|
||||
continue
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
return m
|
||||
return out
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
|
||||
definedIn := map[string]map[string]bool{}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository == assign.SDKRepository {
|
||||
ch.SDKHead = p.Head.Head
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
ch.Members[p.Repository] = p.Head.Head
|
||||
for _, name := range p.Modules {
|
||||
fp, ok := p.Fps[name]
|
||||
if !ok || fp.Manifest == "" {
|
||||
if ok && fp.None2 == noModuleWords {
|
||||
ch.Removed = append(ch.Removed, name)
|
||||
continue
|
||||
}
|
||||
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
|
||||
}
|
||||
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
|
||||
ch.Rebuilt++
|
||||
}
|
||||
got := w.manifest(ctx, manifests, fp.Manifest)
|
||||
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
|
||||
return ch, got.err
|
||||
}
|
||||
ch.Digests[name] = fp.Manifest
|
||||
if got.err != "" {
|
||||
ch.Unreadable[name] = got.err
|
||||
continue
|
||||
}
|
||||
ch.Manifests[got.module] = got.manifest
|
||||
if definedIn[got.module] == nil {
|
||||
definedIn[got.module] = map[string]bool{}
|
||||
}
|
||||
definedIn[got.module][p.Repository] = true
|
||||
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
|
||||
definedIn[got.module][t.Repository] = true
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
}
|
||||
}
|
||||
for name, repos := range definedIn {
|
||||
if len(repos) > 1 {
|
||||
for r := range repos {
|
||||
ch.Twice[name] = append(ch.Twice[name], r)
|
||||
}
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
return ch, ""
|
||||
return out
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
@@ -333,6 +384,10 @@ func (w *world) value(bucket, key string) []byte {
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
|
||||
// manifest but the controller's reads the controller's grants.
|
||||
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
@@ -357,36 +412,14 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
@@ -401,6 +434,14 @@ func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// bucketGen is how many times any key of the bucket was written or deleted in this process: a read of what it shows
|
||||
// elsewhere is asked again only after it moves.
|
||||
func (w *world) bucketGen(bucket string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket]
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
@@ -408,29 +449,35 @@ func (w *world) changed(bucket, key string) uint64 {
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
|
||||
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
|
||||
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
|
||||
_, all := w.assigned()
|
||||
seen := map[string]bool{}
|
||||
for _, a := range all {
|
||||
if a.Module != module || a.Build == "" {
|
||||
continue
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, a.Build))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
|
||||
continue
|
||||
}
|
||||
seen[rec.Commit] = true
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for c := range seen {
|
||||
out = append(out, c)
|
||||
raw, ok := w.read(delivery.HaveBucket)[key]
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil || h.Composed == "" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err == nil && isNode && !placed[node] {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
|
||||
@@ -2,14 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// fakeManifests is the indexer's manifests bucket in memory.
|
||||
@@ -44,76 +42,8 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
|
||||
return p
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
|
||||
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
|
||||
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
|
||||
pull(assign.SDKRepository, 3, nil),
|
||||
}}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
|
||||
if waits != "" {
|
||||
t.Fatalf("the change waits: %s", waits)
|
||||
}
|
||||
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
|
||||
t.Errorf("change = %+v", ch)
|
||||
}
|
||||
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
|
||||
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
|
||||
// saying for what; an unreadable manifest is the change's, not a wait.
|
||||
func TestAChangeWaitsForItsManifests(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
|
||||
for _, c := range []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
|
||||
} {
|
||||
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
|
||||
if waits == "" {
|
||||
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
|
||||
}
|
||||
}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["web"] == "" {
|
||||
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
|
||||
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
|
||||
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
|
||||
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
|
||||
// xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"predefined"`) {
|
||||
t.Fatal("the fixture names no predefined key")
|
||||
}
|
||||
f := fakeManifests{}
|
||||
xterm := f.add(t, "xterm", string(raw))
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
@@ -124,4 +54,27 @@ func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
inPart bool
|
||||
granted bool
|
||||
}{
|
||||
{"the part's grant", true, true},
|
||||
{"no grant", false, false},
|
||||
} {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
|
||||
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
|
||||
}
|
||||
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
|
||||
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 h1:RtAFbUJRP1qO9AIMM2ue+cXNzqj7WZ7IOlSqNvttwuU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -81,6 +81,29 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
@@ -103,6 +126,17 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||
@@ -73,7 +75,13 @@ func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
@@ -87,7 +95,7 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
@@ -97,6 +105,49 @@ func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
// findings bucket cannot be written to at all.
|
||||
func saidConditionsAreFindings(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
raised, cleared asks.Condition
|
||||
key string
|
||||
}{
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
|
||||
f := Findings{Put: func(k string, v []byte) error {
|
||||
got, err := health.DecodeFinding(v)
|
||||
puts[k] = got
|
||||
return err
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(asks.Condition) error { said++; return nil })
|
||||
if err := say(c.raised); err != nil {
|
||||
t.Fatalf("%s: %v", c.key, err)
|
||||
}
|
||||
got, ok := puts[c.key]
|
||||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
|
||||
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
|
||||
}
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
|
||||
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
if err := (Findings{}).Say(func(asks.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
|
||||
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
|
||||
t.Error("a condition not the assigner's is a finding")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
@@ -146,6 +197,44 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
|
||||
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
|
||||
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
|
||||
// and writes nothing itself: the program hands it what it watched and writes what it answers.
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
|
||||
const SDKRepository = "novox/mesh-sdk"
|
||||
|
||||
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
|
||||
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
|
||||
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
|
||||
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
|
||||
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
|
||||
"node-engine's validator would refuse"
|
||||
|
||||
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
|
||||
// why, and each node's open assignments.
|
||||
type Mesh struct {
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Assigned map[string][]string
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
|
||||
// member (empty when none), and how many modules it rebuilds.
|
||||
type Change struct {
|
||||
Group, Heads string
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Removed []string
|
||||
Twice map[string][]string
|
||||
SDKHead string
|
||||
Rebuilt int
|
||||
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine.
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
moved[name] = true
|
||||
}
|
||||
for name := range c.Unreadable {
|
||||
moved[name] = true
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
moved[name] = true
|
||||
}
|
||||
provided, required := map[string]bool{}, map[string]bool{}
|
||||
for name := range moved {
|
||||
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
|
||||
for _, o := range man.Provides {
|
||||
provided[o.Name] = true
|
||||
}
|
||||
}
|
||||
for _, r := range c.Manifests[name].Requires {
|
||||
required[r] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
|
||||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
|
||||
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
|
||||
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
|
||||
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
|
||||
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
|
||||
in := compose.Input{Catalogue: catalog}
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
for _, mod := range assigned[n] {
|
||||
for _, cl := range catalog[mod].Claims {
|
||||
scope := cl.Scope
|
||||
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||
scope = seat.Scope
|
||||
}
|
||||
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
|
||||
}
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
|
||||
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
|
||||
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
|
||||
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
|
||||
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
|
||||
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
|
||||
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
|
||||
var fails []string
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Unreadable) {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
trunk := input(m.Manifests, m.Assigned)
|
||||
changed := map[string]catalogue.Manifest{}
|
||||
for name, man := range m.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for name, man := range c.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
switch {
|
||||
case now == nil:
|
||||
case before != nil:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
|
||||
}
|
||||
}
|
||||
if c.SDKHead != "" {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
|
||||
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
|
||||
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
|
||||
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
|
||||
if c.Rebuilt > delivery.CompositionRebuildWarning {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
|
||||
}
|
||||
v.Warnings = append(v.Warnings, NotJudged)
|
||||
if could := CouldRun(m, c, machines); len(could) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
|
||||
}
|
||||
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
|
||||
}
|
||||
if len(fails) > 0 {
|
||||
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
|
||||
return v
|
||||
}
|
||||
v.State = delivery.CompositionPass
|
||||
return v
|
||||
}
|
||||
|
||||
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
|
||||
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
|
||||
// the facts snapshot), so each is named as not judged.
|
||||
func CouldRun(m Mesh, c Change, composed []string) []string {
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for name := range c.Manifests {
|
||||
if !slices.Contains(modules, name) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
|
||||
func grantsOf(man catalogue.Manifest) string {
|
||||
var offers []string
|
||||
for _, o := range man.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
|
||||
}
|
||||
|
||||
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
|
||||
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
|
||||
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
|
||||
func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
provisions := map[string]bool{}
|
||||
altered := false
|
||||
for name, man := range c.Manifests {
|
||||
before, had := m.Manifests[name]
|
||||
if had && grantsOf(before) == grantsOf(man) {
|
||||
continue
|
||||
}
|
||||
altered = true
|
||||
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
|
||||
provisions[r] = true
|
||||
}
|
||||
}
|
||||
if !altered {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func meshOf() Mesh {
|
||||
return Mesh{
|
||||
Manifests: map[string]catalogue.Manifest{
|
||||
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
|
||||
"web": {Module: "web", Version: "1"},
|
||||
},
|
||||
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
|
||||
}
|
||||
}
|
||||
|
||||
func hasWarning(v delivery.CompositionVerdict, part string) bool {
|
||||
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
|
||||
}
|
||||
|
||||
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
|
||||
// the machine and the mesh-sdk it composed with, and says what it did not judge.
|
||||
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "v0.1.14-x", 0)
|
||||
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
|
||||
}
|
||||
if !hasWarning(v, "issue 1461") {
|
||||
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||
// consumer's machine is touched when its provider changes.
|
||||
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
|
||||
func TestUnreadableTwiceAndWide(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
|
||||
v = Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
|
||||
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
|
||||
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
|
||||
v := Compose(meshOf(), c, "v0.1.14-own", 0)
|
||||
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
|
||||
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
|
||||
}
|
||||
c.SDKHead = ""
|
||||
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
|
||||
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
|
||||
// assigned nowhere touches none, and the verdict says so.
|
||||
func TestTouched(t *testing.T) {
|
||||
m := meshOf()
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
|
||||
t.Errorf("touched by store = %v", got)
|
||||
}
|
||||
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
|
||||
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
|
||||
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
|
||||
// names none of the second.
|
||||
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
|
||||
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
|
||||
m.Assigned["novox"] = []string{"bus"}
|
||||
m.Assigned["shanks"] = []string{"db"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
|
||||
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
|
||||
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
|
||||
}
|
||||
if !hasWarning(v, "whose grants alone the change alters: novox") {
|
||||
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
|
||||
}
|
||||
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
|
||||
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
|
||||
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
|
||||
const (
|
||||
CaseTrunk = "trunk"
|
||||
CaseChange = "change"
|
||||
)
|
||||
|
||||
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
|
||||
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
|
||||
const SDKDir = "go"
|
||||
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
|
||||
// them, the node-engine applies them. Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
|
||||
|
||||
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
|
||||
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
|
||||
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
|
||||
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
|
||||
for name, digest := range m.Digests {
|
||||
trunk[name] = composer.Manifest{Digest: digest}
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for name, digest := range c.Digests {
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
|
||||
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
|
||||
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
|
||||
changed := map[string]composer.Manifest{}
|
||||
for name, digest := range c.Digests {
|
||||
changed[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
|
||||
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
|
||||
}
|
||||
|
||||
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return delivery.ComposeJob{}, err
|
||||
}
|
||||
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
|
||||
Package: composer.Package, Request: raw, Reader: reader}, nil
|
||||
}
|
||||
|
||||
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
|
||||
// composer derives the seats held from the claims, as input does).
|
||||
func plainInput(assigned map[string][]string) compose.Input {
|
||||
var in compose.Input
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
|
||||
type Ran struct {
|
||||
State string
|
||||
Reason string
|
||||
Verdict *composer.Verdict
|
||||
}
|
||||
|
||||
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
|
||||
// failure, named.
|
||||
func RanOf(step delivery.Step) Ran {
|
||||
r := Ran{State: step.State, Reason: step.Reason}
|
||||
if step.State != delivery.StepPassed {
|
||||
return r
|
||||
}
|
||||
var a delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &a); err != nil {
|
||||
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
|
||||
}
|
||||
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
|
||||
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
|
||||
}
|
||||
r.Verdict = &v
|
||||
return r
|
||||
}
|
||||
|
||||
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
|
||||
if v == nil {
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name == name {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
|
||||
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
|
||||
// the group holds one in its repository, else a build its node runs.
|
||||
type Reader struct {
|
||||
Module, Commit string
|
||||
Member bool
|
||||
Ran Ran
|
||||
}
|
||||
|
||||
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
|
||||
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
|
||||
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
|
||||
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
|
||||
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
|
||||
// said and judged nothing.
|
||||
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
var fails []string
|
||||
if head.State != delivery.StepPassed {
|
||||
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
|
||||
v.State = delivery.CompositionFail
|
||||
return v
|
||||
}
|
||||
if head.Verdict.SDK != "" {
|
||||
v.SDK = head.Verdict.SDK
|
||||
}
|
||||
trunk, okT := caseOf(head.Verdict, CaseTrunk)
|
||||
change, okC := caseOf(head.Verdict, CaseChange)
|
||||
if !okT || !okC {
|
||||
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
|
||||
return v
|
||||
}
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if r := change.Manifests[name]; !r.Accepted {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(trunk.Manifests) {
|
||||
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
if _, before := m.Unreadable[name]; before {
|
||||
continue // already unreadable on the trunk: not the group's doing
|
||||
}
|
||||
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
before, now := trunk.Machines[node], change.Machines[node]
|
||||
switch {
|
||||
case now.Composes:
|
||||
case !before.Composes:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
|
||||
}
|
||||
}
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
|
||||
// from before issue 1621): Go's words for a package its module does not hold.
|
||||
func NoComposer(reason string) bool {
|
||||
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
|
||||
if strings.Contains(reason, words) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
|
||||
for _, r := range readers {
|
||||
at := r.Module + " at " + r.Commit
|
||||
if r.Member {
|
||||
at = r.Module + "'s member head " + r.Commit
|
||||
}
|
||||
if r.Ran.State != delivery.StepPassed {
|
||||
if NoComposer(r.Ran.Reason) {
|
||||
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
|
||||
"composer command (before issue 1621)", at))
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
|
||||
continue
|
||||
}
|
||||
read, ok := caseOf(r.Ran.Verdict, CaseChange)
|
||||
if !ok {
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
|
||||
continue
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
|
||||
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
|
||||
}
|
||||
}
|
||||
return fails, warnings
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
const sdkHead = "1111111111111111111111111111111111111111"
|
||||
|
||||
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
|
||||
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
|
||||
var raws = map[string]string{
|
||||
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
|
||||
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
|
||||
"d-web": `{"module": "web", "version": "1"}`,
|
||||
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
|
||||
}
|
||||
|
||||
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
|
||||
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
|
||||
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
|
||||
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
|
||||
|
||||
func digestedMesh() Mesh {
|
||||
m := meshOf()
|
||||
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
|
||||
return m
|
||||
}
|
||||
|
||||
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
|
||||
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
|
||||
func webShares(members map[string]string) Change {
|
||||
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
|
||||
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
|
||||
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
|
||||
}
|
||||
|
||||
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
|
||||
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
|
||||
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
|
||||
t.Helper()
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := composer.Fill(&req, func(d string) ([]byte, error) {
|
||||
if r, ok := read[d]; ok {
|
||||
return []byte(r), nil
|
||||
}
|
||||
return []byte(raws[d]), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.SDK = "(devel) " + job.Fetch["commit"].(string)
|
||||
vr, _ := json.Marshal(v)
|
||||
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
|
||||
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
|
||||
}
|
||||
|
||||
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
|
||||
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
|
||||
// change by digest, never the manifests' bytes, composing the machines the change touches.
|
||||
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, err := HeadJob(digestedMesh(), c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
|
||||
t.Fatalf("job = %+v", job)
|
||||
}
|
||||
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: testTime, Compose: &job}
|
||||
if _, err := delivery.EncodeStepAsk(ask); err != nil {
|
||||
t.Fatalf("the ask is refused by its layout: %v", err)
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
|
||||
t.Fatalf("cases = %+v", req.Cases)
|
||||
}
|
||||
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
|
||||
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
|
||||
}
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
|
||||
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
|
||||
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
v := FromComposer(digestedMesh(), c, head, nil, 0)
|
||||
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
|
||||
t.Fatalf("with the head's composer: %+v", v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
|
||||
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
|
||||
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
|
||||
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
|
||||
// the SDK head (its composer reads what the head reads) it passes.
|
||||
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
running := "3333333333333333333333333333333333333333"
|
||||
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
|
||||
t.Fatalf("reader job = %+v", rjob)
|
||||
}
|
||||
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
|
||||
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
|
||||
}
|
||||
member := "2222222222222222222222222222222222222222"
|
||||
pinned := run(t, rjob, asTheHeadReads)
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
|
||||
if v.State != delivery.CompositionPass {
|
||||
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
|
||||
// failed for another cause fails the group, loud.
|
||||
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
|
||||
t.Fatalf("a reader with no composer: %+v", v)
|
||||
}
|
||||
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
|
||||
t.Fatalf("a reader whose composer failed: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
|
||||
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
|
||||
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
|
||||
t.Fatalf("a head composer that did not run: %+v", v)
|
||||
}
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
|
||||
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
|
||||
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
|
||||
// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair
|
||||
// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its
|
||||
// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router
|
||||
// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the
|
||||
// assigner no longer says it straight (never both).
|
||||
|
||||
// FindingsBucket is the assigner's findings.
|
||||
var FindingsBucket = health.FindingsBucket(delivery.AssignerModule)
|
||||
|
||||
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
|
||||
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
|
||||
// a clear, whose key is to be deleted; value is nil then.
|
||||
func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
|
||||
rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".")
|
||||
if !ok || rest == "" {
|
||||
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
|
||||
}
|
||||
kind, id, _ := strings.Cut(rest, ".")
|
||||
if id == "" {
|
||||
id = c.Subject.ID
|
||||
}
|
||||
scope := c.Subject.Scope
|
||||
if scope == "" {
|
||||
scope = health.ScopeModule
|
||||
}
|
||||
key = health.FindingKey(scope, id, kind)
|
||||
if c.State == asks.StateCleared {
|
||||
return key, nil, false, nil
|
||||
}
|
||||
if c.State != asks.StateRaised {
|
||||
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
|
||||
}
|
||||
summary := c.Summary
|
||||
if summary == "" {
|
||||
summary = c.Headline
|
||||
}
|
||||
var actions []health.Action
|
||||
for _, a := range c.Actions {
|
||||
actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
|
||||
}
|
||||
value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine},
|
||||
Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved,
|
||||
Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind,
|
||||
Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
return key, value, true, err
|
||||
}
|
||||
|
||||
// Findings is where the assigner's findings go: the bucket's put and delete.
|
||||
type Findings struct {
|
||||
Put func(key string, value []byte) error
|
||||
Delete func(key string) error
|
||||
Keys func() ([]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
|
||||
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
|
||||
// findings bucket at all. A finding that cannot be written is the error.
|
||||
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
var errs []error
|
||||
if f.Put != nil {
|
||||
now := time.Now
|
||||
if f.Now != nil {
|
||||
now = f.Now
|
||||
}
|
||||
key, value, raised, err := FindingOf(c, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
errs = append(errs, err)
|
||||
case raised:
|
||||
if err := f.Put(key, value); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
|
||||
}
|
||||
default:
|
||||
if err := f.Delete(key); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.Put == nil {
|
||||
if say != nil {
|
||||
if err := say(c); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
key := delivery.AssignerModule + ".seat-states"
|
||||
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
|
||||
if len(unbound) == 0 {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
|
||||
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
|
||||
|
||||
// PoliciesBucket is the assigner's state `policies`.
|
||||
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
|
||||
|
||||
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
|
||||
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
|
||||
out := map[string]catalogue.PersonPolicy{}
|
||||
var bad []string
|
||||
for key, raw := range values {
|
||||
var p catalogue.PersonPolicy
|
||||
if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key {
|
||||
bad = append(bad, key)
|
||||
continue
|
||||
}
|
||||
out[key] = p
|
||||
}
|
||||
sort.Strings(bad)
|
||||
return out, bad
|
||||
}
|
||||
|
||||
// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one.
|
||||
func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) {
|
||||
var c *catalogue.PersonPolicy
|
||||
if p, ok := chosen[m.Module]; ok {
|
||||
c = &p
|
||||
}
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
|
||||
const KindProvisionFailed = "provision-failed"
|
||||
|
||||
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
|
||||
func ProvisionFailedKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed)
|
||||
}
|
||||
|
||||
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
|
||||
func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
|
||||
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
|
||||
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.",
|
||||
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}},
|
||||
Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func also(provider, consumer string) []string {
|
||||
if provider == "" || provider == consumer {
|
||||
return nil
|
||||
}
|
||||
return []string{provider}
|
||||
}
|
||||
|
||||
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
|
||||
const KindProvisionMissing = "provision-missing"
|
||||
|
||||
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
|
||||
func ProvisionMissingKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
|
||||
}
|
||||
|
||||
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
|
||||
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
|
||||
parts := strings.SplitN(key, ".", 3)
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
|
||||
}
|
||||
node, module, provision := parts[0], parts[1], parts[2]
|
||||
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
|
||||
"no provider of it may be placed", module, node, provision, bound, module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
|
||||
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.",
|
||||
Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6).
|
||||
const KindUnplaced = "unplaced"
|
||||
|
||||
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
|
||||
func UnplacedKey(node string) string {
|
||||
return health.FindingKey(health.ScopeMachine, node, KindUnplaced)
|
||||
}
|
||||
|
||||
// UnplacedFinding is the finding of a node no placement names since the lease write.
|
||||
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+
|
||||
"so the assigner writes it no want", node)
|
||||
return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced,
|
||||
Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary,
|
||||
Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.",
|
||||
Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".",
|
||||
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func clipHeadline(s string) string {
|
||||
if len([]rune(s)) <= 60 {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:60])
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
|
||||
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
|
||||
// the module from its asking until it ends (Module.Puts).
|
||||
|
||||
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
|
||||
// before it is refused.
|
||||
const ComposeWithin = 10 * time.Minute
|
||||
|
||||
// PutWorld is what one step of a put reads.
|
||||
type PutWorld struct {
|
||||
Node, Module string
|
||||
Put delivery.Put
|
||||
// Assigned says the module stands assigned to the node.
|
||||
Assigned bool
|
||||
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
|
||||
Bus, Recorded bool
|
||||
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *delivery.ComposedDigest
|
||||
Granted bool
|
||||
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
|
||||
// that ask expired unanswered.
|
||||
Answer string
|
||||
AskExpired bool
|
||||
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
|
||||
// asked; SnapshotFailed why it was not taken.
|
||||
Snapshot string
|
||||
Snapshotting bool
|
||||
SnapshotFailed string
|
||||
// Have is what the node says it runs of the module; nil when it says nothing.
|
||||
Have *delivery.Have
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
|
||||
type PutNext struct {
|
||||
Put delivery.Put
|
||||
// Ask writes the put's ask about the move (PutAsk).
|
||||
Ask bool
|
||||
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
|
||||
Snapshot bool
|
||||
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
|
||||
Warranted bool
|
||||
}
|
||||
|
||||
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
|
||||
func StepPut(w PutWorld) PutNext {
|
||||
p := w.Put
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return PutNext{Put: p}
|
||||
}
|
||||
say := func(step, format string, args ...any) PutNext {
|
||||
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
}
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
|
||||
}
|
||||
since := w.Now
|
||||
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
|
||||
since = p.Steps[len(p.Steps)-1].At
|
||||
}
|
||||
if w.Now.Sub(since) > ComposeWithin {
|
||||
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
|
||||
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
|
||||
}
|
||||
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
|
||||
}
|
||||
d := *w.Composed
|
||||
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
|
||||
return PutNext{Put: applied(w, p)}
|
||||
}
|
||||
p.Composed = d.Digest
|
||||
if needsPerson(w, d) {
|
||||
switch w.Answer {
|
||||
case OptionNo:
|
||||
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
|
||||
case OptionYes:
|
||||
default:
|
||||
if w.AskExpired {
|
||||
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
|
||||
}
|
||||
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
|
||||
if p.Ask == "" {
|
||||
next.Ask = true
|
||||
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
|
||||
}
|
||||
return next
|
||||
}
|
||||
if w.Bus {
|
||||
switch {
|
||||
case w.Snapshot != "":
|
||||
p.Snapshot = w.Snapshot
|
||||
case w.SnapshotFailed != "":
|
||||
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
|
||||
case w.Snapshotting:
|
||||
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
default:
|
||||
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
next.Snapshot = true
|
||||
return next
|
||||
}
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
if p.Snapshot != "" {
|
||||
words += "; the streams were snapshotted: " + p.Snapshot
|
||||
}
|
||||
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
|
||||
next.Warranted = w.Bus && p.Ask != ""
|
||||
return next
|
||||
}
|
||||
|
||||
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
|
||||
func applied(w PutWorld, p delivery.Put) delivery.Put {
|
||||
h := w.Have
|
||||
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
|
||||
return p
|
||||
}
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
|
||||
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
|
||||
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
|
||||
case failed(*h):
|
||||
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
|
||||
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
|
||||
// container, move data, or that cannot be judged (§14).
|
||||
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
|
||||
runs := w.Have != nil && w.Have.FP == w.Put.FP
|
||||
if runs && w.Have.Composed == d.Digest {
|
||||
return false // the node runs this composition already: nothing moves
|
||||
}
|
||||
if w.Bus {
|
||||
return !(runs && d.UserListOnly)
|
||||
}
|
||||
if !w.Recorded {
|
||||
return false
|
||||
}
|
||||
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||
func PutWant(p delivery.Put) delivery.Want {
|
||||
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
|
||||
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
|
||||
switch p.Step {
|
||||
case delivery.PutWanted, delivery.PutHolding:
|
||||
w := PutWant(p)
|
||||
return &w, true
|
||||
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
|
||||
return nil, false
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
|
||||
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
|
||||
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
|
||||
func PutAsking(p delivery.Put) int {
|
||||
if n := p.At.UnixMilli(); n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||
func PutAct(node, module, fp, composed string) asks.Act {
|
||||
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
}
|
||||
|
||||
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + d.Words,
|
||||
delivery.JudgedNeither: "recreates no container and moves no data",
|
||||
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||
}[d.Judgement]
|
||||
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||
expires := now.Add(7 * 24 * time.Hour)
|
||||
if bus {
|
||||
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
yes.Level, yes.Binds = asks.Approve, binds
|
||||
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||
expires = now.Add(asks.ApproveLasts)
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, PutAsking(p)),
|
||||
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||
Explanation: explanation,
|
||||
Options: []asks.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||
Who: asks.Operator,
|
||||
Expires: expires,
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}, Asking: PutAsking(p)}, nil
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
|
||||
Warrant *delivery.AddNodeWarrant
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
}
|
||||
if p.Why == "" {
|
||||
findings = append(findings, "a proposal says why: it is shown to the operator")
|
||||
}
|
||||
if len(in.Changes) == 0 {
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
}
|
||||
out := map[string]*delivery.Placement{}
|
||||
for node, pl := range laid {
|
||||
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
|
||||
continue
|
||||
}
|
||||
pl := pl
|
||||
all[node] = pl
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
|
||||
// candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
|
||||
for _, node := range p.RemovedNodes() {
|
||||
if _, ok := in.Current[node]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(all, node)
|
||||
out[node] = nil
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
held, recorded := in.Held, in.Recorded
|
||||
if held == nil {
|
||||
held = func(string) bool { return false }
|
||||
}
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
// A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by
|
||||
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
|
||||
joining := map[string]bool{}
|
||||
for _, c := range in.Changes {
|
||||
if c.Op == delivery.OpAddNode {
|
||||
joining[c.Node] = true
|
||||
}
|
||||
}
|
||||
if len(joining) > 0 {
|
||||
was := recorded
|
||||
recorded = func(node string) bool { return joining[node] || was(node) }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// ControllerModule is the controller's module, which nothing places again (issue 1126).
|
||||
const ControllerModule = "mesh-controller"
|
||||
|
||||
// ControllerEnded is why a placement of the controller is refused: the parts that acted only once its lease was handed
|
||||
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
|
||||
// would write beside them, silently.
|
||||
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
|
||||
|
||||
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
|
||||
// whole placement, or in what the changes lay. None when nothing does.
|
||||
func ControllerRefusals(changes []delivery.PlacementProposalChange, laid map[string]delivery.Placement) []string {
|
||||
var out []string
|
||||
nodes := map[string]bool{}
|
||||
for i, c := range changes {
|
||||
places := c.Module == ControllerModule && c.Op != delivery.OpTakeOff
|
||||
if c.Whole != nil && slices.Contains(c.Whole.Assigned(), ControllerModule) {
|
||||
places = true
|
||||
}
|
||||
if places {
|
||||
out = append(out, fmt.Sprintf("change %d places %s on %s: %s", i+1, ControllerModule, c.Node, ControllerEnded))
|
||||
nodes[c.Node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(laid)) {
|
||||
if !nodes[node] && slices.Contains(laid[node].Assigned(), ControllerModule) {
|
||||
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
|
||||
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
|
||||
// it is created) and its value (nil: it is deleted).
|
||||
type Write struct {
|
||||
Node, Key string
|
||||
Base uint64
|
||||
Value *delivery.Placement
|
||||
}
|
||||
|
||||
// Writes is every key a proposal writes, by node order.
|
||||
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
|
||||
var out []Write
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
|
||||
// the controller's verbs (ADR 0567 §8, issue 774).
|
||||
const (
|
||||
KindControllerCall = "controller-call"
|
||||
KindControllerPin = "controller-pin"
|
||||
)
|
||||
|
||||
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
|
||||
// module, or node and provision for a pin).
|
||||
func CallClearedCondition(kind, node, module string) asks.Condition {
|
||||
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
|
||||
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
|
||||
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
func PlacementAct(p delivery.PlacementProposal) asks.Act {
|
||||
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
|
||||
}
|
||||
|
||||
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
|
||||
// another asking, under an id of its own.
|
||||
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
|
||||
return int(at.Unix())
|
||||
}
|
||||
|
||||
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
|
||||
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
|
||||
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
|
||||
binds, err := asks.ActDigest(PlacementAct(p))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: headline,
|
||||
Explanation: brief,
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
|
||||
Level: asks.Approve, Binds: binds},
|
||||
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts),
|
||||
OnExpiry: "nothing is written, and the proposal ends",
|
||||
About: key,
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
rest := p
|
||||
rest.Changes, rest.Base = nil, map[string]uint64{}
|
||||
for _, c := range p.Changes {
|
||||
if _, done := written[c.Node]; !done {
|
||||
rest.Changes = append(rest.Changes, c)
|
||||
}
|
||||
}
|
||||
for node, base := range p.Base {
|
||||
if _, done := written[node]; !done {
|
||||
rest.Base[node] = base
|
||||
}
|
||||
}
|
||||
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
|
||||
return rest, len(rest.Changes) > 0
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
|
||||
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
|
||||
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
|
||||
func TestThePlacementSeatDecides(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
|
||||
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
|
||||
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
|
||||
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
|
||||
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
|
||||
Held: func(m string) bool { return m != "ghost" },
|
||||
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
|
||||
}
|
||||
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
|
||||
p, laid, findings := Propose(in(move...))
|
||||
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
|
||||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
|
||||
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
|
||||
}
|
||||
if _, err := delivery.EncodePlacementProposal(p); err != nil {
|
||||
t.Fatalf("the proposal cannot be written: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
|
||||
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
|
||||
// act without its lease.
|
||||
"the controller placed again": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
|
||||
"the controller seeded whole": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "anchor", Whole: &delivery.Placement{V: delivery.Version, Node: "anchor",
|
||||
Decided: anchor.Decided, Modules: []delivery.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("%s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
|
||||
t.Errorf("the rest = %+v, %v", rest, ok)
|
||||
}
|
||||
if _, ok := Rest(p, laid); ok {
|
||||
t.Error("a proposal written whole leaves something to write")
|
||||
}
|
||||
|
||||
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
|
||||
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
|
||||
c := SeatWaitsCondition(unbound)
|
||||
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
|
||||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
|
||||
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
|
||||
}
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
q := p
|
||||
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||
@@ -28,13 +30,29 @@ type Module struct {
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
|
||||
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
|
||||
Provisions func(node string) map[string]provisions.Provision
|
||||
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
||||
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
||||
Answering func(provision string) []catalogue.Manifest
|
||||
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
|
||||
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
|
||||
// is waited on for want of an offer.
|
||||
Offered func(provision string) bool
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
|
||||
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
|
||||
// approved release is known to carry it. Nil: none is read.
|
||||
Release func(fp string) (string, bool)
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
@@ -44,8 +62,16 @@ type Module struct {
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
||||
Self func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
@@ -80,14 +106,15 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
|
||||
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3).
|
||||
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
|
||||
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
|
||||
// down and one module never blocks another.
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
|
||||
// without which there is nothing to want: the build done, not held, the grant and the composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
@@ -103,11 +130,6 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
@@ -117,11 +139,86 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
||||
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
||||
func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
if m.Provisions == nil {
|
||||
return "", false
|
||||
}
|
||||
if missing := m.MissingProvisions(node); len(missing) > 0 {
|
||||
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
||||
}
|
||||
keyed := m.Provisions(node)
|
||||
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
|
||||
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
|
||||
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
|
||||
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
|
||||
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
|
||||
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
|
||||
// bound (ADR 0603 §8).
|
||||
needed := m.Manifest.Requires
|
||||
if m.Answering != nil {
|
||||
needed = catalogue.Awaited(m.Manifest, m.Answering)
|
||||
have := map[string]bool{}
|
||||
for _, p := range keyed {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
|
||||
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
|
||||
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
|
||||
continue
|
||||
}
|
||||
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
|
||||
continue
|
||||
}
|
||||
needed = append(needed, r)
|
||||
if !have[r] {
|
||||
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
||||
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
|
||||
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
||||
if p.Why != "" {
|
||||
words += ": " + p.Why
|
||||
}
|
||||
return words, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
|
||||
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
|
||||
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
|
||||
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
|
||||
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
|
||||
func (m Module) MissingProvisions(node string) []string {
|
||||
if m.Provisions == nil || m.Answering == nil {
|
||||
return nil
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, p := range m.Provisions(node) {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
var out []string
|
||||
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
|
||||
if !have[r] {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
||||
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
||||
// offers it) and holds.
|
||||
@@ -148,6 +245,17 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
||||
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
||||
}
|
||||
|
||||
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
|
||||
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
|
||||
func unreached(m Module, node string, n Nodes) string {
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return why
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
||||
// valuable or rebuildable; cache and none do not).
|
||||
func keepsData(m catalogue.Manifest) bool {
|
||||
@@ -167,6 +275,9 @@ type Outcome struct {
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
@@ -174,6 +285,9 @@ type Outcome struct {
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
|
||||
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
|
||||
Unreached map[string]string
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
@@ -190,12 +304,18 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
|
||||
// §2): no bar of its own.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words.
|
||||
func (h Holding) Reason() string {
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
@@ -209,11 +329,25 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.FP != m.FP || !failed(h) {
|
||||
continue
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
@@ -224,6 +358,29 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
p, put := m.Puts[node]
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||
// without hold is not kept this way: its put takes it further once heard.
|
||||
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||
Step: delivery.PutHolding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
if want != nil {
|
||||
w := *want
|
||||
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
||||
w.Composed = ul.Composed
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
@@ -235,26 +392,40 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
|
||||
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
|
||||
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
|
||||
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
|
||||
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
|
||||
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
|
||||
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
|
||||
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
if older {
|
||||
_, composed := m.composition(node, have.FP)
|
||||
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
|
||||
}
|
||||
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
switch {
|
||||
case older:
|
||||
ok, target = true, have.FP
|
||||
case oldForm:
|
||||
// held back by rule 3 as written: on neither list until the canary runs F healthy
|
||||
default:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
}
|
||||
}
|
||||
if why := unreached(m, node, n); why != "" {
|
||||
if out.Unreached == nil {
|
||||
out.Unreached = map[string]string{}
|
||||
}
|
||||
out.Unreached[node] = why
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
||||
n.Haves[node][m.Name].FP != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
@@ -266,7 +437,8 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && before.FP == target && before.Composed == d.Digest {
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -308,8 +480,18 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
|
||||
h, there := n.Haves[node][m.Name]
|
||||
switch {
|
||||
case there && h.Phase != delivery.PhaseRemoved && !held:
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
||||
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||
if !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -333,6 +515,10 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
}
|
||||
}
|
||||
next, write := m.selfDerived(node, before, had, next, n, &out)
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
@@ -345,6 +531,48 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
|
||||
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
||||
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
||||
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
||||
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
||||
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
||||
// while its ask waits, the want written stays and the node's holding says why.
|
||||
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
|
||||
switch {
|
||||
case next.Removed:
|
||||
return next, true
|
||||
case m.selfOn(node):
|
||||
next.Self, next.Composed = true, ""
|
||||
return next, true
|
||||
case !next.Self && next.Composed != "":
|
||||
return next, true
|
||||
}
|
||||
next.Self, next.Yes = false, ""
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
at.FP = next.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
return next, false
|
||||
}
|
||||
next.Composed = d.Digest
|
||||
return next, true
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
@@ -354,6 +582,43 @@ func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
@@ -411,10 +676,10 @@ func short(fp string) string {
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
|
||||
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
|
||||
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
|
||||
// streams before it applies (§2); the assigner asks no snapshot for it.
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
@@ -426,14 +691,6 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
@@ -443,9 +700,14 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
|
||||
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
||||
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if m.Chosen != nil {
|
||||
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
@@ -475,17 +737,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node: its own `<node>.<module>` or a process's
|
||||
// `<node>.<module>.<process>`. A grant that names no account (a module that does not speak on the bus) names every node.
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
if len(g.Accounts) == 0 {
|
||||
return true
|
||||
}
|
||||
own := node + "." + module
|
||||
for _, a := range g.Accounts {
|
||||
if a == own || strings.HasPrefix(a, own+".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.Grant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
@@ -65,13 +66,16 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
delivery.RuleCanary: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
},
|
||||
delivery.RuleProviders: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
},
|
||||
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
||||
},
|
||||
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
|
||||
delivery.RuleProvisions: func(m *Module, _ *Nodes) {
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
|
||||
}
|
||||
},
|
||||
} {
|
||||
m, n := ready()
|
||||
breakIt(&m, &n)
|
||||
@@ -85,12 +89,250 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
|
||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||
}
|
||||
if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
|
||||
t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
|
||||
for _, all := range []func(string) map[string]provisions.Provision{
|
||||
func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}}
|
||||
}, nil} {
|
||||
m, n := ready()
|
||||
m.Provisions = all
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew {
|
||||
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision
|
||||
// asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided
|
||||
// oidc moves the want, an asked one holds it and names oidc and never database.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
state, older string
|
||||
moves bool
|
||||
}{{"the replaced edge provided, the older edge retiring", provisions.Provided, provisions.Retiring, true},
|
||||
{"the replaced edge provided, the older edge asked", provisions.Provided, provisions.Asked, true},
|
||||
{"the replaced edge asked, the older edge retiring", provisions.Asked, provisions.Retiring, false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{"oidc"}
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
|
||||
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
|
||||
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
|
||||
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
|
||||
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
|
||||
{
|
||||
handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}}
|
||||
keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}}
|
||||
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh,
|
||||
AnsweredBy: catalogue.AnsweredByHelper}}}
|
||||
store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}}
|
||||
goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}}
|
||||
gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}}
|
||||
answering := func(p string) []catalogue.Manifest {
|
||||
for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} {
|
||||
if m.Provides[0].Name == p {
|
||||
return []catalogue.Manifest{m}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
manifest catalogue.Manifest
|
||||
missing string // the provision named, "" for none: the want moves
|
||||
}{
|
||||
{"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"database"}}, "database"},
|
||||
{"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""},
|
||||
{"a module not handed waits for its route, which the helper answers (grafana's route)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"},
|
||||
{"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)",
|
||||
catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""},
|
||||
{"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Manifest, m.Answering = c.manifest, answering
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if c.missing == "" {
|
||||
if out.Wants["g14"].FP != fNew || len(m.MissingProvisions("g14")) != 0 {
|
||||
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out.Rules["g14"].Rule != delivery.RuleProvisions || !strings.Contains(out.Rules["g14"].Words,
|
||||
"provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 {
|
||||
t.Errorf("%s: %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
}
|
||||
// ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own
|
||||
// name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not.
|
||||
for _, c := range []struct {
|
||||
name, requires string
|
||||
offered bool
|
||||
}{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true},
|
||||
{"a provision nobody offers", "database", false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{c.requires}
|
||||
m.Answering = func(string) []catalogue.Manifest { return nil }
|
||||
m.Offered = func(p string) bool { return c.offered && p == c.requires }
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != delivery.RuleProvisions ||
|
||||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
|
||||
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
|
||||
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
|
||||
// healthy held photos, and novox offline left a module with no canary.
|
||||
for _, c := range []struct {
|
||||
name, module, provision, provider string
|
||||
have *delivery.Have // what novox runs of the module; nil: a new assignment
|
||||
before map[string]delivery.Want
|
||||
online bool
|
||||
want string
|
||||
}{
|
||||
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
|
||||
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
|
||||
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
|
||||
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
|
||||
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
|
||||
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
|
||||
} {
|
||||
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
|
||||
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
|
||||
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
|
||||
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
|
||||
if c.have != nil {
|
||||
n.Haves["novox"][c.module] = *c.have
|
||||
}
|
||||
out := Wants(m, n, c.before, nil)
|
||||
w, written := out.Wants["novox"]
|
||||
if !written {
|
||||
w = c.before["novox"]
|
||||
}
|
||||
if w.FP != c.want || w.Removed {
|
||||
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
|
||||
}
|
||||
if _, on := out.Neither["novox"]; on {
|
||||
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
|
||||
}
|
||||
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
|
||||
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
|
||||
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
|
||||
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
|
||||
// stays and is said; a placed node's want is Self with no composition.
|
||||
selfAt := func(fp string) map[string]delivery.Want {
|
||||
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
|
||||
}
|
||||
atF := func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
set func(*Module, *Nodes)
|
||||
before string
|
||||
want delivery.Want // the want of g14 written; zero: none written
|
||||
rule int // the rule g14's holding names
|
||||
asked bool
|
||||
}{
|
||||
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
|
||||
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
|
||||
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
|
||||
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
|
||||
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
|
||||
rule: delivery.RuleNotHeld},
|
||||
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Held = true
|
||||
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
|
||||
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
|
||||
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
recording(m)
|
||||
m.Held = true
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
|
||||
}
|
||||
}, rule: delivery.RuleNotHeld, asked: true},
|
||||
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, rule: delivery.RuleComposed},
|
||||
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Self = func(node string) bool { return node == "g14" }
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, want: delivery.Want{FP: fNew, Self: true}},
|
||||
} {
|
||||
m, n := ready()
|
||||
c.set(&m, &n)
|
||||
before := selfAt(c.before)
|
||||
if c.name == "placed" {
|
||||
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
}
|
||||
out := Wants(m, n, before, nil)
|
||||
if got := out.Wants["g14"]; got != c.want {
|
||||
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
|
||||
}
|
||||
if got := out.Rules["g14"].Rule; got != c.rule {
|
||||
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
|
||||
}
|
||||
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
|
||||
t.Errorf("%s: record asks %+v", c.name, out.Record)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok {
|
||||
if _, err := delivery.EncodeWant(w); err != nil {
|
||||
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
grant := delivery.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
@@ -109,7 +351,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
if got := GrantNamesNode(delivery.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
@@ -150,6 +392,72 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
||||
}
|
||||
|
||||
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
|
||||
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
|
||||
if out.Held != nil || len(out.Wants) != 0 {
|
||||
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
|
||||
}
|
||||
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
|
||||
len(out.NodeFailed) != 1 {
|
||||
t.Errorf("node failed = %+v", out.NodeFailed)
|
||||
}
|
||||
|
||||
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
|
||||
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
|
||||
three := m
|
||||
three.Nodes = []string{"ace", "g14", "novox"}
|
||||
failedHave := n.Haves["ace"]["mail"]
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
|
||||
n.Online["novox"] = true
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
|
||||
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if _, alone := out.NodeFailed["g14"]; !alone {
|
||||
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
|
||||
}
|
||||
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
|
||||
}
|
||||
if _, written := out.Wants["g14"]; written {
|
||||
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
|
||||
}
|
||||
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
delete(before, "novox")
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
|
||||
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
|
||||
}
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
|
||||
|
||||
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
|
||||
// not one node's failure (D3). Either way F is held and ace is set back.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
online map[string]bool
|
||||
novox delivery.Health
|
||||
}{
|
||||
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
|
||||
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
|
||||
} {
|
||||
n.Online = c.online
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
|
||||
if c.novox == delivery.HealthOK {
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
|
||||
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld {
|
||||
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
||||
@@ -370,11 +678,13 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
||||
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
||||
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
||||
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
|
||||
// backs its streams up before it applies.
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
|
||||
t.Errorf("the bus did not move by the rules: %+v", out)
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
@@ -384,6 +694,142 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||
}
|
||||
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
|
||||
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
|
||||
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
wanted map[string]delivery.Want
|
||||
want *delivery.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
|
||||
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
} {
|
||||
p := put
|
||||
p.Step = c.step
|
||||
m.Puts = map[string]delivery.Put{"ace": p}
|
||||
if c.step == "" {
|
||||
m.Puts = nil
|
||||
}
|
||||
out := Wants(m, n, c.wanted, nil)
|
||||
got, wrote := out.Wants["ace"]
|
||||
switch {
|
||||
case c.want == nil && wrote:
|
||||
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
|
||||
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
|
||||
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
|
||||
}
|
||||
}
|
||||
userList := "sha256:" + strings.Repeat("7", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
held := put
|
||||
held.Step = delivery.PutHolding
|
||||
m.Puts = map[string]delivery.Put{"ace": held}
|
||||
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
|
||||
// operator's yes, the bus's snapshot, the want, and the have.
|
||||
func stepsOfAPut(t *testing.T) {
|
||||
t.Helper()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
|
||||
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
world func() PutWorld
|
||||
step string
|
||||
says string
|
||||
ask, snap bool
|
||||
warranted bool
|
||||
}{
|
||||
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
|
||||
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
|
||||
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
|
||||
{"none within the bound: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Composed = nil
|
||||
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
|
||||
return w
|
||||
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
|
||||
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
|
||||
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
|
||||
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
|
||||
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
|
||||
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
|
||||
{"snapshotted: the want, warranted", func() PutWorld {
|
||||
w := bus()
|
||||
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
|
||||
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
|
||||
{"a user-list revision of the bus needs nobody", func() PutWorld {
|
||||
w := bus()
|
||||
ul := d
|
||||
ul.UserListOnly = true
|
||||
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released", false, false, false},
|
||||
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
|
||||
{"a recorded module judged to recreate asks", func() PutWorld {
|
||||
w := bus()
|
||||
w.Bus, w.Recorded = false, true
|
||||
r := d
|
||||
r.Judgement = delivery.JudgedRecreates
|
||||
w.Composed = &r
|
||||
return w
|
||||
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"wanted, the node runs it: holding", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutHolding, "held there until released", false, false, false},
|
||||
{"wanted without hold: done", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put.Hold = false
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutDone, "the rules write its want again", false, false, false},
|
||||
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
|
||||
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
|
||||
c.ask, c.snap, c.warranted)
|
||||
}
|
||||
if next.Ask && next.Put.Ask == "" {
|
||||
t.Errorf("put, %s: an ask without its key", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||
@@ -393,6 +839,13 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
|
||||
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
|
||||
m.Self = func(string) bool { return true }
|
||||
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer in a placed node's want was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
@@ -452,10 +905,16 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
||||
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
||||
}
|
||||
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
|
||||
with(delivery.PolicyRecord, "person")
|
||||
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
||||
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
||||
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
|
||||
// bus has no bar of its own (ADR 0601 §1).
|
||||
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
@@ -464,8 +923,8 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
||||
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -653,10 +1112,12 @@ func x11() (Module, Nodes) {
|
||||
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
out := Wants(m, n, nil, nil)
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||
}
|
||||
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
|
||||
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
|
||||
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
|
||||
}
|
||||
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
|
||||
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
|
||||
}
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("left on neither list: %+v", out.Neither)
|
||||
@@ -694,11 +1155,22 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
if what == "an old-form build" {
|
||||
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("%s: shanks's want = %+v, want F", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("%s: shanks on neither list = %+v", what, h)
|
||||
}
|
||||
}
|
||||
m, n := x11()
|
||||
@@ -707,10 +1179,12 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||
}
|
||||
for node, w := range out.Wants {
|
||||
if w.FP != fOld {
|
||||
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||
}
|
||||
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
|
||||
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
|
||||
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
|
||||
}
|
||||
if w := out.Wants["g14"]; w.FP != fNew {
|
||||
t.Errorf("the canary g14's want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -721,16 +1195,44 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
m, n := x11()
|
||||
delete(n.Haves["shanks"], "screen-lock")
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
|
||||
}
|
||||
m.Built = false
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||
}
|
||||
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
|
||||
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
|
||||
// runs the module is on neither list, its removed want left as it is.
|
||||
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before map[string]delivery.Want
|
||||
want map[string]delivery.Want
|
||||
}{
|
||||
{"no key: nothing written", nil, map[string]delivery.Want{}},
|
||||
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
|
||||
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
|
||||
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
|
||||
map[string]delivery.Want{}},
|
||||
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
|
||||
} {
|
||||
out := Wants(m, n, c.before, nil)
|
||||
if len(out.Wants) != len(c.want) {
|
||||
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
|
||||
continue
|
||||
}
|
||||
for node, w := range c.want {
|
||||
if !delivery.SameWant(normal(out.Wants[node]), w) {
|
||||
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.FP = ""
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||
@@ -740,7 +1242,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
oldForm := func(n Nodes, node string) {
|
||||
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
@@ -774,16 +1276,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||
}
|
||||
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
|
||||
m, n = x11()
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
|
||||
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
|
||||
}
|
||||
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||
if _, on := out.Neither["shanks"]; on {
|
||||
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
|
||||
}
|
||||
// F not granted: nothing, even with the canary healthy
|
||||
m, n = x11()
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
// Package judge is how the seat mesh-placement's holder judges a proposal (novox/hq ADR 0567 §3, §4, as ADR 0593 §8
|
||||
// changes them): what it changes on each node by its placement (Intended), every node answering it, and the verdict of
|
||||
// their answers (JudgeAnswers). The assigner's own (ADR 0593 §1), moved back from mesh-sdk's compose/judge: the holder
|
||||
// composes nothing, since every composition is its node's engine's, and a node answers through the candidates path
|
||||
// with what it composed. It holds no bus client and no timer: the holder gives it what it read.
|
||||
package judge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// AnswerBound is how long the holder waits for a node's answer after its candidate was written: the time to live of the
|
||||
// node's pulse key (delivery.PulseAnswerKey), whose expiry names it as giving no answer, never passed. The same answer
|
||||
// bound a check gives a node (ADR 0593 §8, §7: 60 seconds, delivery.ComposeAnswerBound). A variable so a test on the bus
|
||||
// can shorten it.
|
||||
var AnswerBound = delivery.ComposeAnswerBound
|
||||
|
||||
// Intended is, by node, the modules whose placed entry the proposal changes there, and the nodes that answer it: every
|
||||
// node (ADR 0593 §8, changing ADR 0567 §4's list of the nodes that must answer), sorted. current is the placement as it
|
||||
// stands, laid the proposal's placements by node (nil removes the node's key) and nodes every node with a record. A
|
||||
// node's set holds the modules the proposal adds, removes or whose placed entry differs on that node; a node with no
|
||||
// key now gets an empty set (§7: the seed is answered as the same). What a node's compositions change beyond its own
|
||||
// placement it says itself in its answer, marked in the brief as not computed.
|
||||
func Intended(current map[string]delivery.Placement, laid map[string]*delivery.Placement, nodes []string) (intended map[string]map[string]bool, must []string) {
|
||||
intended = map[string]map[string]bool{}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n] = true
|
||||
}
|
||||
for n := range laid {
|
||||
all[n] = true
|
||||
}
|
||||
for node := range all {
|
||||
must = append(must, node)
|
||||
set := map[string]bool{}
|
||||
intended[node] = set
|
||||
before, placed := current[node]
|
||||
after, changesKey := laid[node]
|
||||
if !changesKey || !placed {
|
||||
continue // untouched, or new to placement: every answer must be the same
|
||||
}
|
||||
var afterModules []delivery.PlacedModule
|
||||
if after != nil {
|
||||
afterModules = after.Modules
|
||||
}
|
||||
entries := map[string]delivery.PlacedModule{}
|
||||
for _, m := range before.Modules {
|
||||
entries[m.Module] = m
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range afterModules {
|
||||
seen[m.Module] = true
|
||||
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
|
||||
set[m.Module] = true
|
||||
}
|
||||
}
|
||||
for name := range entries {
|
||||
if !seen[name] {
|
||||
set[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(must)
|
||||
return intended, must
|
||||
}
|
||||
|
||||
// JudgeAnswers is the verdict of a proposal's answers (§4). answers is by node its HaveNode.Candidates entry for the
|
||||
// proposal (nil when it has none) and revisions by node the revision of its candidate key: an answer counts when it
|
||||
// names the proposal and that revision. It fails on a refusal by a node that must answer; a node of Must with no answer
|
||||
// that counts is waited for while its pulse key stands, and once it has ended (pulsing false: the bound's expiry, heard
|
||||
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
|
||||
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
|
||||
// shown in the ask's brief. The seed alone fails on any difference.
|
||||
func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
|
||||
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) delivery.PlacementVerdict {
|
||||
v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
|
||||
Answers: map[string]delivery.CandidateAnswer{}}
|
||||
for node, set := range intended {
|
||||
for m := range set {
|
||||
v.Intended[node] = append(v.Intended[node], m)
|
||||
}
|
||||
sort.Strings(v.Intended[node])
|
||||
}
|
||||
must := p.Must
|
||||
if len(must) == 0 {
|
||||
must = p.Nodes()
|
||||
}
|
||||
for _, node := range must {
|
||||
a := answers[node]
|
||||
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
|
||||
v.Silent = append(v.Silent, node)
|
||||
continue
|
||||
}
|
||||
v.Answers[node] = *a
|
||||
if a.Refused != "" {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s cannot compose the proposal: %s", node, a.Refused))
|
||||
continue
|
||||
}
|
||||
for _, r := range a.Refusals {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s refuses the proposal for %s: %s", node, r.Module, r.Why))
|
||||
}
|
||||
want := intended[node]
|
||||
differs := func(module, words string) {
|
||||
if p.Seed {
|
||||
// The seed carries what its nodes found, each named in the brief for the operator who approves it.
|
||||
v.Carried[node] = append(v.Carried[node], words)
|
||||
return
|
||||
}
|
||||
if !want[module] {
|
||||
v.Unintended[node] = append(v.Unintended[node], module)
|
||||
}
|
||||
}
|
||||
for _, d := range a.Differs {
|
||||
differs(d.Module, differenceWords(d))
|
||||
}
|
||||
for _, d := range a.Added {
|
||||
differs(d.Module, "would run "+d.Module)
|
||||
}
|
||||
for _, m := range a.Removed {
|
||||
differs(m, "would stop running "+m)
|
||||
}
|
||||
sort.Strings(v.Unintended[node])
|
||||
}
|
||||
if len(v.Intended) == 0 {
|
||||
v.Intended = nil
|
||||
}
|
||||
if len(v.Unintended) == 0 {
|
||||
v.Unintended = nil
|
||||
}
|
||||
if len(v.Carried) == 0 {
|
||||
v.Carried = nil
|
||||
}
|
||||
if len(v.Answers) == 0 {
|
||||
v.Answers = nil
|
||||
}
|
||||
var waiting, gone []string
|
||||
for _, node := range v.Silent {
|
||||
named := node
|
||||
if online != nil && !online[node] {
|
||||
named += " (offline)"
|
||||
}
|
||||
if pulsing[node] {
|
||||
waiting = append(waiting, named)
|
||||
} else {
|
||||
gone = append(gone, named)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("waits for the answers of %s, each for %s from its candidate's write",
|
||||
strings.Join(waiting, ", "), AnswerBound))
|
||||
}
|
||||
if len(gone) > 0 {
|
||||
// ADR 0593 §8: decided on the answers it has; that node's own composition judges again when it applies (§9).
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("no answer from %s within %s of its candidate's write: decided on the answers "+
|
||||
"it has, and that node's own composition judges again when it applies", strings.Join(gone, ", "), AnswerBound))
|
||||
}
|
||||
switch {
|
||||
case len(v.Fails) > 0:
|
||||
v.State = delivery.VerdictFail
|
||||
case len(waiting) > 0:
|
||||
v.State = delivery.VerdictWaits
|
||||
default:
|
||||
v.State = delivery.VerdictPass
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Brief is the verdict as the ask's brief shows it, by node (§5): what the proposal changes and what each node answered
|
||||
// beside it, marked, and what stays held as the node holds it today, with the proposal's why.
|
||||
func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s (proposed by %s)", p.Why, p.By)
|
||||
nodes := map[string]bool{}
|
||||
for n := range v.Intended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range v.Unintended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for _, n := range p.Nodes() {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n, a := range v.Answers {
|
||||
if len(a.Held) > 0 {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
for n := range v.Carried {
|
||||
nodes[n] = true
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, "; %s:", n)
|
||||
if m := v.Intended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, " changes %s", strings.Join(m, ", "))
|
||||
} else {
|
||||
b.WriteString(" composes the same")
|
||||
}
|
||||
if m := v.Unintended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; also differs, not computed: %s", strings.Join(m, ", "))
|
||||
}
|
||||
if m := v.Carried[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; the seed carries: %s", strings.Join(m, "; "))
|
||||
}
|
||||
if held := v.Answers[n].Held; len(held) > 0 {
|
||||
var names []string
|
||||
for _, h := range held {
|
||||
today := h.Today
|
||||
if today == "" {
|
||||
today = "held"
|
||||
}
|
||||
names = append(names, h.Module+" ("+today+")")
|
||||
}
|
||||
fmt.Fprintf(&b, "; stays as today: %s", strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
|
||||
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
|
||||
// field), and how many fields more; or the resource and why when no body is shown.
|
||||
func differenceWords(d delivery.ModuleDiffers) string {
|
||||
for _, b := range d.Bodies {
|
||||
if len(b.Paths) == 0 {
|
||||
continue
|
||||
}
|
||||
path := b.Paths[0]
|
||||
words := fmt.Sprintf("%s: %s %s → %s", d.Module, fieldName(b.Running, b.Composed, path),
|
||||
valueAt(b.Running, path), valueAt(b.Composed, path))
|
||||
if n := len(b.Paths) - 1; n > 0 {
|
||||
words += fmt.Sprintf(" (and %d more fields)", n)
|
||||
}
|
||||
return words
|
||||
}
|
||||
if d.Resource != "" {
|
||||
return fmt.Sprintf("%s: %s (%s)", d.Module, d.Resource, d.Why)
|
||||
}
|
||||
return fmt.Sprintf("%s: %s", d.Module, d.Why)
|
||||
}
|
||||
|
||||
// shownValue bounds a value named in a brief.
|
||||
const shownValue = 60
|
||||
|
||||
// walk follows a differing path (delivery.DifferingPaths: keys joined by dots, a key may hold dots, and a JSON value
|
||||
// written in a string is read as JSON) through a body, answering the keys it passed and the value at its end.
|
||||
func walk(body, path string) (keys []string, value any, ok bool) {
|
||||
var v any
|
||||
if json.Unmarshal([]byte(body), &v) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest := path
|
||||
for rest != "" {
|
||||
if s, isString := v.(string); isString {
|
||||
var inner any
|
||||
if json.Unmarshal([]byte(s), &inner) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
v = inner
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
best := ""
|
||||
for k := range t {
|
||||
if (rest == k || strings.HasPrefix(rest, k+".")) && len(k) > len(best) {
|
||||
best = k
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, best), t[best], strings.TrimPrefix(strings.TrimPrefix(rest, best), ".")
|
||||
case []any:
|
||||
seg, after, _ := strings.Cut(rest, ".")
|
||||
n, err := strconv.Atoi(seg)
|
||||
if err != nil || n < 0 || n >= len(t) {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, seg), t[n], after
|
||||
default:
|
||||
return nil, nil, false
|
||||
}
|
||||
}
|
||||
return keys, v, true
|
||||
}
|
||||
|
||||
// valueAt is the value at path in a shown body, cut to shownValue, or why it is not shown.
|
||||
func valueAt(body, path string) string {
|
||||
if body == "" {
|
||||
return "(none)"
|
||||
}
|
||||
_, v, ok := walk(body, path)
|
||||
if !ok {
|
||||
return "(value cut)"
|
||||
}
|
||||
s, isString := v.(string)
|
||||
if !isString {
|
||||
raw, _ := json.Marshal(v)
|
||||
s = string(raw)
|
||||
}
|
||||
if r := []rune(s); len(r) > shownValue {
|
||||
s = string(r[:shownValue-1]) + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fieldName is the differing field as a person reads it: its parent's name and its own, `PARENT[key]`, from whichever
|
||||
// body reaches it; the path's end when neither does.
|
||||
func fieldName(running, composed, path string) string {
|
||||
for _, body := range []string{running, composed} {
|
||||
if keys, _, ok := walk(body, path); ok && len(keys) > 0 {
|
||||
if len(keys) == 1 {
|
||||
return keys[0]
|
||||
}
|
||||
return keys[len(keys)-2] + "[" + keys[len(keys)-1] + "]"
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package judge
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §3, §4 (rows moved from the assigner's check of a pull request's placement, ADR 0559 §5): what a proposal
|
||||
// changes by node, and the verdict of the answers: differences shown and failing nothing, a refusal and a silent node
|
||||
// failing, a later answer judging again, the seed failing on any difference.
|
||||
func TestJudge(t *testing.T) {
|
||||
current := map[string]delivery.Placement{"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "server": {Node: "server"}}
|
||||
laid := map[string]*delivery.Placement{
|
||||
"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}},
|
||||
"laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: "web"}}}}
|
||||
// ADR 0593 §8: every node answers; what the placement changes is read from the placement alone.
|
||||
got, must := Intended(current, laid, []string{"anchor", "laptop", "server"})
|
||||
if !got["anchor"]["mail"] || !got["anchor"]["web"] || len(got["laptop"]) != 0 || len(got["server"]) != 0 ||
|
||||
!slices.Equal(must, []string{"anchor", "laptop", "server"}) {
|
||||
t.Errorf("intended = %v, must %v: want anchor's mail and web, nothing elsewhere, every node answering", got, must)
|
||||
}
|
||||
|
||||
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
|
||||
p := delivery.PlacementProposal{ID: "p-one", By: "op", Why: "move web", Written: at, Base: map[string]uint64{"anchor": 3, "laptop": 0},
|
||||
Must: []string{"anchor", "laptop"}}
|
||||
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
|
||||
intended := map[string]map[string]bool{"anchor": {"web": true}}
|
||||
same := func(rev uint64) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
|
||||
}
|
||||
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
seed bool
|
||||
answers map[string]*delivery.CandidateAnswer
|
||||
pulsing map[string]bool
|
||||
state string
|
||||
words string
|
||||
marked string
|
||||
}{
|
||||
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
|
||||
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
|
||||
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
|
||||
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
|
||||
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, delivery.VerdictWaits, "laptop (offline)", ""},
|
||||
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
|
||||
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
} {
|
||||
q := p
|
||||
q.Seed = row.seed
|
||||
v := JudgeAnswers(q, intended, row.answers, revisions, map[string]bool{"anchor": true}, row.pulsing, at)
|
||||
all := strings.Join(append(append([]string{}, v.Fails...), v.Notes...), "; ")
|
||||
if v.State != row.state || !strings.Contains(all, row.words) {
|
||||
t.Errorf("%s: %s, %q", row.name, v.State, all)
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass && !strings.Contains(Brief(q, v), "the seed carries: node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → (value cut) (and 1 more fields)") {
|
||||
t.Errorf("%s: the brief does not name it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass {
|
||||
whole := row.answers["laptop"].Differs[0]
|
||||
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
|
||||
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
|
||||
t.Errorf("the difference is named %q", got)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(row.name, "a module held as today") && !strings.Contains(Brief(q, v), "stays as today: spiegel (held)") {
|
||||
t.Errorf("%s: the brief does not show it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.marked != "" && (!slices.Contains(v.Unintended["anchor"], row.marked) || !strings.Contains(Brief(q, v), "not computed: "+row.marked)) {
|
||||
t.Errorf("%s: the difference is not marked: %+v, %s", row.name, v.Unintended, Brief(q, v))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func load(t *testing.T) catalogue.Manifest {
|
||||
@@ -15,16 +16,16 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
|
||||
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -32,9 +33,11 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
|
||||
// every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1).
|
||||
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
|
||||
"mesh-provisioner.provisions", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -42,7 +45,8 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -51,12 +55,41 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
// 1906/D3: it says its conditions on the operator channel; it asks the Go build seat nothing since the composition is
|
||||
// the node-engine's (ADR 0593 §7).
|
||||
if !slices.Equal(m.Uses, []string{"operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly operator-channel", m.Uses)
|
||||
}
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its five verbs (adopt and hand-back went with the
|
||||
// controller, ADR 0590); keeps the seat's events on its stream placement for 30 days; calls the hand-act log; and reads
|
||||
// mesh-config's resolved values to compose every node a proposal touches.
|
||||
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
|
||||
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again"}) ||
|
||||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its five verbs and mesh-wants with its two", m.Claims)
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
|
||||
}
|
||||
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
t.Errorf("emits = %v, want put", m.Emits)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,29 +2,82 @@
|
||||
"module": "mesh-delivery-assigner",
|
||||
"version": "1",
|
||||
"slug": "assigner",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-placement",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"placement",
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mesh-wants",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"upgrade",
|
||||
"bus"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-grants.grants",
|
||||
"mesh-host.composed",
|
||||
"mesh-provisioner.provisions",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:node-backup.now"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
],
|
||||
"tools": [
|
||||
"assigner_put",
|
||||
"assigner_release",
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "placement", "history": 64 },
|
||||
{ "name": "proposals", "history": 5 },
|
||||
{ "name": "candidates", "history": 5 },
|
||||
{ "name": "pulses", "key-ttl": true },
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 },
|
||||
{ "name": "findings", "history": 5 },
|
||||
{ "name": "policies", "history": 20 },
|
||||
{ "name": "seat-holders", "history": 5 }
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "placement",
|
||||
"events": [
|
||||
"placement-proposed",
|
||||
"placement-judged",
|
||||
"placement-changed",
|
||||
"placement-withdrawn"
|
||||
],
|
||||
"max-age-days": 30
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
@@ -71,7 +124,10 @@
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-assigner",
|
||||
"binary": "mesh-delivery-assigner"
|
||||
"binary": "mesh-delivery-assigner",
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
|
||||
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
|
||||
// and the asker reads the first such key that is not running. The builder and the checker share it.
|
||||
|
||||
// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
|
||||
type AskPublisher interface {
|
||||
PublishAsk(ctx context.Context, subject, id string, data []byte) error
|
||||
}
|
||||
|
||||
// AskWaiter watches an ask's steps: every value a key `*.<ask id>` of the build module's steps state takes, running or
|
||||
// ended, on the channel it answers, until ctx ends. A watch, not a poll.
|
||||
type AskWaiter interface {
|
||||
WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error)
|
||||
}
|
||||
|
||||
// Asker puts asks and waits for their steps.
|
||||
type Asker struct {
|
||||
Publisher AskPublisher
|
||||
Waiter AskWaiter
|
||||
// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
|
||||
// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
|
||||
// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Put puts a on its stream: the seat's accept subject of its language and kind.
|
||||
func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error {
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil {
|
||||
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
|
||||
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
|
||||
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
|
||||
func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
watch, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
var bound, outerCap <-chan time.Time
|
||||
var timer, capTimer *time.Timer
|
||||
defer func() {
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
}()
|
||||
running, since, phase, phaseAt := "", time.Time{}, "", time.Time{}
|
||||
standStill := func() error {
|
||||
if phase == "" {
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound)
|
||||
}
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s",
|
||||
a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.Step{}, ctx.Err()
|
||||
case <-bound:
|
||||
return delivery.Step{}, standStill()
|
||||
case <-outerCap:
|
||||
return delivery.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended",
|
||||
a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
case step, ok := <-steps:
|
||||
if !ok {
|
||||
return delivery.Step{}, errors.New("the watch of the ask's step ended")
|
||||
}
|
||||
if seen := stepSeenOf(ctx); seen != nil {
|
||||
seen(step)
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return step, nil
|
||||
}
|
||||
if k.Bound <= 0 {
|
||||
continue
|
||||
}
|
||||
newRun := step.Node != running || !step.Started.Equal(since)
|
||||
if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) {
|
||||
continue
|
||||
}
|
||||
// A node took the ask, or took it again after its run stopped (another node, or the same node's build module
|
||||
// replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the
|
||||
// step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to
|
||||
// three bounds from its start (issue 1930).
|
||||
running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt
|
||||
last := step.Started
|
||||
if step.PhaseAt.After(last) {
|
||||
last = step.PhaseAt
|
||||
}
|
||||
left := k.Bound
|
||||
if !last.IsZero() {
|
||||
left = time.Until(last.Add(k.Bound))
|
||||
}
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
timer = time.NewTimer(max(left, 0))
|
||||
bound = timer.C
|
||||
if newRun {
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
outer := 3 * k.Bound
|
||||
if !step.Started.IsZero() {
|
||||
outer = time.Until(step.Started.Add(3 * k.Bound))
|
||||
}
|
||||
capTimer = time.NewTimer(max(outer, 0))
|
||||
outerCap = capTimer.C
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type stepSeenKey struct{}
|
||||
|
||||
// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node
|
||||
// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so
|
||||
// (issue 1890). seen runs in the waiting goroutine and must not block.
|
||||
func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context {
|
||||
return context.WithValue(ctx, stepSeenKey{}, seen)
|
||||
}
|
||||
|
||||
func stepSeenOf(ctx context.Context) func(delivery.Step) {
|
||||
seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step))
|
||||
return seen
|
||||
}
|
||||
|
||||
// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
|
||||
func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
return k.Passed(k.Wait(ctx, a))
|
||||
}
|
||||
|
||||
// Passed is step when it passed, else an error naming where it ended and why.
|
||||
func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) {
|
||||
if err != nil {
|
||||
return step, err
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
|
||||
}
|
||||
return step, nil
|
||||
}
|
||||
|
||||
// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
|
||||
// of `*.<ask id>` in the build module's steps bucket.
|
||||
type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
|
||||
msg.Header.Set(jetstream.MsgIDHeader, id)
|
||||
_, err := s.JS.PublishMsg(ctx, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
|
||||
// once, and every change after.
|
||||
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
w, err := kv.Watch(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
|
||||
}
|
||||
select {
|
||||
case out <- step:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
|
||||
// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for
|
||||
// concurrent use.
|
||||
type FakeAsks struct {
|
||||
Answer func(a delivery.StepAsk) delivery.Step
|
||||
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
steps map[string][]delivery.Step
|
||||
watchers map[string][]chan delivery.Step
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.asks = append(f.asks, a)
|
||||
f.mu.Unlock()
|
||||
if f.Answer != nil {
|
||||
f.Write(f.Answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write writes an ask's step, as a build node does.
|
||||
func (f *FakeAsks) Write(step delivery.Step) {
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
f.steps[step.ID] = append(f.steps[step.ID], step)
|
||||
ws := append([]chan delivery.Step(nil), f.watchers[step.ID]...)
|
||||
f.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- step
|
||||
}
|
||||
}
|
||||
|
||||
// End is Write: kept for the tests that end a step.
|
||||
func (f *FakeAsks) End(step delivery.Step) { f.Write(step) }
|
||||
|
||||
// WatchAsk is AskWaiter's: what was written so far, then every write.
|
||||
func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan delivery.Step, error) {
|
||||
out := make(chan delivery.Step, 64)
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
for _, s := range f.steps[askID] {
|
||||
out <- s
|
||||
}
|
||||
f.watchers[askID] = append(f.watchers[askID], out)
|
||||
f.mu.Unlock()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
f.mu.Lock()
|
||||
ws := f.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == out {
|
||||
f.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Asks are the asks put so far.
|
||||
func (f *FakeAsks) Asks() []delivery.StepAsk {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), f.asks...)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build integration
|
||||
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// 1887/A4a, 1887/AC3 on the real bus: an ask put twice under one id is one message, and the asker reads the step a node
|
||||
// wrote under `<node>.<ask id>`.
|
||||
func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Subjects: []string{"mesh.seat.node-build-go.accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket("mesh-build-go")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus := JetStreamAsks{JS: js}
|
||||
k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
|
||||
a := goAsk("mvl0frv18d1960")
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
s, _ := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
||||
if info, _ := s.Info(ctx); info.State.Msgs != 1 {
|
||||
t.Fatalf("%d messages for one ask put twice", info.State.Msgs)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
running, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
_, _ = kv.Put(ctx, key, running)
|
||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepPassed, Started: time.Now(),
|
||||
Result: json.RawMessage(`{"artifacts":[]}`)})
|
||||
_, _ = kv.Put(ctx, key, passed)
|
||||
}()
|
||||
step, err := k.Passed(k.Wait(ctx, a))
|
||||
if err != nil || step.Node != "ace" {
|
||||
t.Fatalf("step %+v, %v", step, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func goAsk(id string) delivery.StepAsk {
|
||||
return delivery.StepAsk{ID: id, Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now(),
|
||||
Build: &delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}}
|
||||
}
|
||||
|
||||
// 1887/A4a: an asker puts the ask once and answers the step a build node wrote for it, naming the node; one that did not
|
||||
// pass is an error naming the node and why; a wait of an ask already put puts nothing.
|
||||
func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
f := &FakeAsks{Answer: func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)}
|
||||
}}
|
||||
k := Asker{Publisher: f, Waiter: f}
|
||||
var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it
|
||||
step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957"))
|
||||
if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" {
|
||||
t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen)
|
||||
}
|
||||
f.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"}
|
||||
}
|
||||
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
|
||||
t.Fatalf("a refused step: %v", err)
|
||||
}
|
||||
// review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step
|
||||
// running, the step is bounded from its start, and the error names the node.
|
||||
g := &FakeAsks{}
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
|
||||
waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded {
|
||||
t.Fatalf("an ask no node took: %v, want it still waiting when ctx ended", err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1961", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1961")); err == nil || !strings.Contains(err.Error(), "on ace did not end within") {
|
||||
t.Fatalf("a step that never ends: %v", err)
|
||||
}
|
||||
if len(g.Asks()) != 0 {
|
||||
t.Fatal("a wait put an ask")
|
||||
}
|
||||
// Issue 1930: the bound is on a step that stands still, not on its age. A step past one bound from its start that entered a
|
||||
// new phase inside the bound is waited for until it ends; one that stood a whole bound in a phase fails at once, naming
|
||||
// the phase; one three bounds from its start ends, said as kept moving. The steps' own times stand for the clock, so the
|
||||
// bound is an hour no loaded machine reaches and no case waits on a pause (the check of 2026-11-02 22:35 tripped one
|
||||
// of 60 ms).
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: time.Hour}
|
||||
now := time.Now()
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepRunning, Started: now.Add(-90 * time.Minute), Phase: "pb", PhaseAt: now})
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepPassed})
|
||||
if step, err := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || step.State != delivery.StepPassed {
|
||||
t.Fatalf("a step that kept moving past its bound: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1981", Node: "ace", State: delivery.StepRunning, Started: now.Add(-time.Hour), Phase: "integration: tests", PhaseAt: now.Add(-time.Hour)})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1981")); err == nil || !strings.Contains(err.Error(), `stood still in "integration: tests"`) {
|
||||
t.Fatalf("a step that stood still in a phase: %v", err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1982", Node: "ace", State: delivery.StepRunning, Started: now.Add(-3 * time.Hour), Phase: "qz", PhaseAt: now})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1982")); err == nil || !strings.Contains(err.Error(), "kept moving") {
|
||||
t.Fatalf("a step that never stops moving: %v", err)
|
||||
}
|
||||
// Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed
|
||||
// with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written
|
||||
// first is answered as the node wrote it.
|
||||
dead := &DeadLetters{}
|
||||
lw := LostAskWaiter{Inner: g, Dead: dead}
|
||||
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
|
||||
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
|
||||
if !ok {
|
||||
t.Fatal("a dead letter's headers were not read")
|
||||
}
|
||||
go dead.Heard(given)
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970"))
|
||||
if err != nil || step.State != delivery.StepFailed || !IsLost(step) ||
|
||||
!strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") {
|
||||
t.Fatalf("a given-up ask: %+v, %v", step, err)
|
||||
}
|
||||
if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) {
|
||||
t.Fatalf("a given-up ask's error: %v", err)
|
||||
}
|
||||
step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971"))
|
||||
if err != nil || !IsNotTaken(step) || IsLost(step) {
|
||||
t.Fatalf("an ask no node took: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed})
|
||||
if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" {
|
||||
t.Fatalf("a step written: %+v, %v", step, err)
|
||||
}
|
||||
// Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and
|
||||
// tries again on the next ask waited for; heard, it says so and hands the given-up asks on.
|
||||
fol := &refusingFollower{refuse: 1}
|
||||
var mu sync.Mutex
|
||||
var told []Hearing
|
||||
hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil,
|
||||
func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() })
|
||||
hearing.Wait()
|
||||
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
|
||||
t.Fatalf("a refused hearing says %q", why)
|
||||
}
|
||||
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hearing.Wait()
|
||||
mu.Lock()
|
||||
if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 {
|
||||
t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls())
|
||||
}
|
||||
mu.Unlock()
|
||||
fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5})
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974"))
|
||||
if err != nil || !IsLost(step) {
|
||||
t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err)
|
||||
}
|
||||
// The lost check of 2026-10-28: a step whose own watch went deaf is read at the next nudge (a step of its build module
|
||||
// ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again.
|
||||
d := &FakeAsks{}
|
||||
heard := &HeardEnds{Inner: deafWatch{}, Read: d}
|
||||
k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute}
|
||||
d.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "shanks", State: delivery.StepPassed})
|
||||
got := make(chan delivery.Step, 1)
|
||||
go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }()
|
||||
for heard.Waiting() == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
heard.Nudge(NodeBuildModule)
|
||||
select {
|
||||
case s := <-got:
|
||||
t.Fatalf("a nudge of another build module answered %+v", s)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
heard.Nudge(GoBuildModule)
|
||||
if s := <-got; s.State != delivery.StepPassed || s.Node != "shanks" {
|
||||
t.Fatalf("a step its watch never handed over: %+v", s)
|
||||
}
|
||||
closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d}
|
||||
if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != delivery.StepPassed {
|
||||
t.Fatalf("a watch that closed once: %+v, %v", s, err)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
h *Hearing
|
||||
want string
|
||||
}{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} {
|
||||
var raw []byte
|
||||
if row.h != nil {
|
||||
raw, _ = json.Marshal(row.h)
|
||||
}
|
||||
if a := HearingHealth(raw); a.State != row.want || a.Why == "" {
|
||||
t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the
|
||||
// new run's start, not the first run's. The steps' own times stand for the clock: the first run started now, the second a
|
||||
// whole bound ago, so the wait ends at once on the second run's bound where the first run's would wait an hour; no case
|
||||
// waits on a pause a loaded machine can outlast (the check of 2026-11-02 22:35 tripped one of 60 ms). The context's
|
||||
// bound is reached only by a wait that kept the first run's start.
|
||||
func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
|
||||
f := &FakeAsks{}
|
||||
k := Asker{Publisher: f, Waiter: f, Bound: time.Hour}
|
||||
const id = "mvl0frv18d1962"
|
||||
now := time.Now()
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: now})
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: now.Add(-time.Hour)})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := k.Wait(ctx, goAsk(id)); err == nil || !strings.Contains(err.Error(), "on ace did not end within 1h0m0s of its start") {
|
||||
t.Fatalf("a redelivered run a bound old: %v, want its own bound to end the wait", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed.
|
||||
type refusingFollower struct {
|
||||
mu sync.Mutex
|
||||
refuse int
|
||||
n int
|
||||
hand func(GivenUp)
|
||||
}
|
||||
|
||||
func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.n++
|
||||
if f.n <= f.refuse {
|
||||
return nil, errors.New("permissions violation")
|
||||
}
|
||||
f.hand = heard
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n }
|
||||
|
||||
func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) }
|
||||
|
||||
// deafWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type deafWatch struct{}
|
||||
|
||||
func (deafWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
// closingWatch closes its first watch at once, and hands the later ones to inner.
|
||||
type closingWatch struct {
|
||||
inner AskWaiter
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (c *closingWatch) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if !c.closed {
|
||||
c.closed = true
|
||||
ch := make(chan delivery.Step)
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
return c.inner.WatchAsk(ctx, module, askID)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
|
||||
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
|
||||
// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
|
||||
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
|
||||
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
|
||||
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
|
||||
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
|
||||
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
|
||||
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
|
||||
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
|
||||
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
|
||||
tc, none := delivery.AgreedToolchain(entries)
|
||||
if none != "" {
|
||||
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
|
||||
}
|
||||
nodes, err := choose(language, entries, places, qualifies, want)
|
||||
return nodes, tc, err
|
||||
}
|
||||
|
||||
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
|
||||
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
|
||||
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
return choose(language, entries, places, qualifies, want)
|
||||
}
|
||||
|
||||
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
if places == nil {
|
||||
places = &Places{}
|
||||
}
|
||||
nodes := make([]string, 0, len(entries))
|
||||
var reporting []string
|
||||
seen := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if seen[e.Node] {
|
||||
continue
|
||||
}
|
||||
seen[e.Node] = true
|
||||
reporting = append(reporting, e.Node)
|
||||
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node
|
||||
nodes = append(nodes, e.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
sort.Strings(reporting)
|
||||
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
|
||||
}
|
||||
return places.choose(nodes, want), nil
|
||||
}
|
||||
|
||||
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
|
||||
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
|
||||
// building again settles nothing; it waits for another fingerprint.
|
||||
const DigestsDifferPrefix = "digests differ:"
|
||||
|
||||
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
|
||||
// digest when they differ. One node's digest agrees with itself.
|
||||
func Agree(artifact string, byNode map[string]string) (string, error) {
|
||||
nodes := make([]string, 0, len(byNode))
|
||||
for n := range byNode {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var first string
|
||||
for i, n := range nodes {
|
||||
if i == 0 {
|
||||
first = byNode[n]
|
||||
continue
|
||||
}
|
||||
if byNode[n] != first {
|
||||
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
|
||||
for _, n := range nodes {
|
||||
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
|
||||
}
|
||||
return "", fmt.Errorf("%s", msg[:len(msg)-1])
|
||||
}
|
||||
}
|
||||
return first, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
|
||||
func hex64(c byte) string { return strings.Repeat(string(c), 64) }
|
||||
|
||||
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
|
||||
var out []delivery.NodeToolchain
|
||||
for _, n := range nodes {
|
||||
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
|
||||
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
|
||||
}
|
||||
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
|
||||
t.Errorf("toolchain = %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
|
||||
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
|
||||
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
|
||||
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
|
||||
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
|
||||
}
|
||||
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
|
||||
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
|
||||
}
|
||||
}
|
||||
|
||||
func agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
|
||||
if err != nil || d != digest {
|
||||
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
|
||||
}
|
||||
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
|
||||
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
|
||||
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
|
||||
}
|
||||
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
|
||||
t.Errorf("one node: %q, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
|
||||
online := func(n string) bool { return n != "g14" } // a sleeping laptop
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
|
||||
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
|
||||
}
|
||||
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
|
||||
}
|
||||
if tc.Version != "go1.27.1" {
|
||||
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// placesRunning is Places with these steps counted as running, for a test of the choice.
|
||||
func placesRunning(running map[string]int) *Places {
|
||||
return &Places{running: running, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func choosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
|
||||
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
|
||||
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
|
||||
p.Capacity = map[string]int{"shanks": 3}
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The choice of the nodes a step runs on and the agreement of their answers, one table of cases (the test budget).
|
||||
func TestChoosingTheNodesOfAStep(t *testing.T) {
|
||||
t.Run("choose takes the agreed nodes with fewest running steps ties by name", chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName)
|
||||
t.Run("choose with one node answers one node", chooseWithOneNodeAnswersOneNode)
|
||||
t.Run("choose fails naming the language and the versions when no node qualifies", chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies)
|
||||
t.Run("agree needs equal digests and names both nodes when they differ", agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer)
|
||||
t.Run("choose skips nodes that are not online and fails naming them when none is", chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs)
|
||||
t.Run("choose prefers nodes below their capacity", choosePrefersNodesBelowTheirCapacity)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps
|
||||
// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the
|
||||
// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the
|
||||
// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484).
|
||||
package buildrun
|
||||
@@ -0,0 +1,22 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 h1:RtAFbUJRP1qO9AIMM2ue+cXNzqj7WZ7IOlSqNvttwuU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,56 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
|
||||
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
|
||||
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
|
||||
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
|
||||
// and from a build at once.
|
||||
type HaveSet struct {
|
||||
mu sync.Mutex
|
||||
healthy map[string]bool // by `<node>.<module>`
|
||||
}
|
||||
|
||||
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
|
||||
// (`<node>.node`) and a value that is not a have are left out.
|
||||
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return false
|
||||
}
|
||||
ok := false
|
||||
if !deleted {
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
||||
}
|
||||
k := node + "." + module
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.healthy[k] == ok {
|
||||
return false
|
||||
}
|
||||
if s.healthy == nil {
|
||||
s.healthy = map[string]bool{}
|
||||
}
|
||||
if ok {
|
||||
s.healthy[k] = true
|
||||
} else {
|
||||
delete(s.healthy, k)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Healthy is whether the module on the node was last seen running and healthy.
|
||||
func (s *HaveSet) Healthy(node, module string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.healthy[node+"."+module]
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A step's end is never only its own watch's to hear** (the lost check of 2026-10-28: mesh-catalog #553's verdict
|
||||
// stayed running after its step mvsgthyjed5786 had passed on shanks at 19:09:31Z; the watch of that one key, opened at
|
||||
// 18:46:05Z, heard neither the step's start nor its end and never closed, while a watch opened at 18:58:48Z heard its own
|
||||
// step end). HeardEnds stands between the asker (Asker) and the watch of each ask's step key: when it is
|
||||
// nudged, every ask it waits on reads its step keys now (a read, not the watch) and hands what it reads to its wait, so a
|
||||
// step that ended is judged even when its own watch went deaf. A watch that closes under a wait is opened again, at most
|
||||
// reopenTries times. It is nudged on an event (Nudge): any step of the same build module ending (one watch of each steps
|
||||
// state, HearEnds), a reconnect to the bus, and whatever else the asker hears (the checker: checker_ask_again).
|
||||
// It holds no timer. Every asker of the build seats waits through it (the checker and the builder, HearEnds).
|
||||
|
||||
// StepReader reads, now, every step a build module's steps state holds for an ask: its keys `*.<ask id>`.
|
||||
type StepReader interface {
|
||||
AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error)
|
||||
}
|
||||
|
||||
// reopenTries is how often a wait opens its ask's watch again after it closed, before the wait ends failed.
|
||||
const reopenTries = 3
|
||||
|
||||
// HeardEnds is an AskWaiter over Inner that also reads an ask's steps when nudged. Safe for concurrent use.
|
||||
type HeardEnds struct {
|
||||
Inner AskWaiter
|
||||
Read StepReader
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
waiting map[string]*waitedAsk // by ask id
|
||||
}
|
||||
|
||||
type waitedAsk struct {
|
||||
module string
|
||||
nudge chan struct{}
|
||||
}
|
||||
|
||||
func (h *HeardEnds) logf(format string, args ...any) {
|
||||
if h.Logf != nil {
|
||||
h.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: what Inner's watch hands, and, at each nudge, what the ask's keys hold now.
|
||||
func (h *HeardEnds) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
in, err := h.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &waitedAsk{module: module, nudge: make(chan struct{}, 1)}
|
||||
h.mu.Lock()
|
||||
if h.waiting == nil {
|
||||
h.waiting = map[string]*waitedAsk{}
|
||||
}
|
||||
h.waiting[askID] = w
|
||||
h.mu.Unlock()
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
if h.waiting[askID] == w {
|
||||
delete(h.waiting, askID)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
reopened := 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-in:
|
||||
if !ok {
|
||||
if ctx.Err() != nil || reopened >= reopenTries {
|
||||
h.logf("the watch of the ask %s's step on %s closed %d times; its wait ends", askID, module, reopened+1)
|
||||
return
|
||||
}
|
||||
reopened++
|
||||
if in, err = h.Inner.WatchAsk(ctx, module, askID); err != nil {
|
||||
h.logf("the watch of the ask %s's step on %s closed and cannot be opened again, so its wait ends: %v", askID, module, err)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the ask %s's step on %s closed; opened again (%d of %d)", askID, module, reopened, reopenTries)
|
||||
continue
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case <-w.nudge:
|
||||
if h.Read == nil {
|
||||
continue
|
||||
}
|
||||
steps, err := h.Read.AskSteps(ctx, module, askID)
|
||||
if err != nil {
|
||||
h.logf("the steps of the ask %s on %s cannot be read: %v", askID, module, err)
|
||||
continue
|
||||
}
|
||||
for _, s := range steps {
|
||||
if s.State != delivery.StepRunning {
|
||||
h.logf("the ask %s's step on %s ended %s, read at a nudge: its watch had not handed it over", askID, s.Node, s.State)
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Nudge has every ask waited on of module read its steps now; module "" nudges every ask.
|
||||
func (h *HeardEnds) Nudge(module string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for _, w := range h.waiting {
|
||||
if module == "" || w.module == module {
|
||||
select {
|
||||
case w.nudge <- struct{}{}:
|
||||
default: // a nudge is pending already
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Waiting is how many asks are waited on now.
|
||||
func (h *HeardEnds) Waiting() int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return len(h.waiting)
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the keys `*.<ask id>` of the build module's steps state, read now.
|
||||
func (s JetStreamAsks) AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
lister, err := kv.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []delivery.Step
|
||||
for key := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue // deleted between the list and the read
|
||||
}
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
out = append(out, step)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HearEnds is the HeardEnds every asker of the build seats waits through: over inner (the asks' own watches, or a
|
||||
// LostAskWaiter's inner), reading with asks, and nudged at each step end of each of modules (one watch of each steps state,
|
||||
// updates only; one that closes is opened again, and said and left after reopenTries closes in a row that heard nothing). The asker nudges it at a reconnect too (Nudge("")).
|
||||
func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules []string, logf func(string, ...any)) *HeardEnds {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf}
|
||||
for _, module := range modules {
|
||||
kv, err := asks.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
|
||||
continue
|
||||
}
|
||||
go h.follow(ctx, kv, module)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) {
|
||||
silent := 0 // closes in a row having heard nothing
|
||||
for ctx.Err() == nil {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
h.logf("the steps of %s cannot be watched, so an ask whose own watch goes deaf is read only at a reconnect: %v", module, err)
|
||||
return
|
||||
}
|
||||
heard := false
|
||||
for e := range w.Updates() {
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
heard = true
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil && step.State != delivery.StepRunning {
|
||||
h.Nudge(module)
|
||||
}
|
||||
}
|
||||
_ = w.Stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
h.Nudge(module) // a step may have ended while the watch was closing
|
||||
if heard {
|
||||
silent = 0
|
||||
} else if silent++; silent >= reopenTries {
|
||||
h.logf("the watch of the steps of %s closed %d times in a row having heard nothing, so it is left: an ask whose own watch goes deaf is read only at a reconnect", module, silent)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the steps of %s closed; opened again", module)
|
||||
}
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the last step written for the ask, if any.
|
||||
func (f *FakeAsks) AskSteps(_ context.Context, _ string, askID string) ([]delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if s := f.steps[askID]; len(s) > 0 {
|
||||
return []delivery.Step{s[len(s)-1]}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask
|
||||
// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node
|
||||
// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under
|
||||
// mesh.events.dead.<seat stream>.<worker>, with the ask's message id (its ask id), how often it was handed out and when it
|
||||
// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays
|
||||
// running and holds a try place, a check's verdict stays running and its pull request waits.
|
||||
//
|
||||
// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on
|
||||
// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose
|
||||
// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its
|
||||
// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the
|
||||
// builder asks its part of a build again).
|
||||
//
|
||||
// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took
|
||||
// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise
|
||||
// (ADR 0525 §9): a backlog is no fault.
|
||||
|
||||
// LostNode is the node a step ended for a lost ask names: no build node wrote it.
|
||||
const LostNode = "bus"
|
||||
|
||||
// The words of a step ended for its ask.
|
||||
const (
|
||||
LostWords = "its ask was given up by the build seat"
|
||||
NotTakenWords = "no build node took its ask"
|
||||
)
|
||||
|
||||
// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link
|
||||
// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up.
|
||||
const (
|
||||
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
|
||||
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
|
||||
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
|
||||
DeadHeaderStream = "Mesh-Dead-Stream"
|
||||
)
|
||||
|
||||
// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat
|
||||
// stream's copies there (buscompose DeadLetterSubject).
|
||||
const DeadLettersStream = "DEAD_LETTERS"
|
||||
|
||||
// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under.
|
||||
func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" }
|
||||
|
||||
// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-<language>).
|
||||
func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) }
|
||||
|
||||
// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it.
|
||||
type GivenUp struct {
|
||||
AskID string
|
||||
Stream string
|
||||
Deliveries int
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id.
|
||||
func GivenUpOf(header map[string][]string) (GivenUp, bool) {
|
||||
get := func(k string) string {
|
||||
if v := header[k]; len(v) > 0 {
|
||||
return strings.TrimSpace(v[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)}
|
||||
if g.AskID == "" {
|
||||
return GivenUp{}, false
|
||||
}
|
||||
g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries))
|
||||
g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp))
|
||||
return g, true
|
||||
}
|
||||
|
||||
// Words are what a step ended for the given-up ask says.
|
||||
func (g GivenUp) Words() string {
|
||||
at := "at an unknown time"
|
||||
if !g.At.IsZero() {
|
||||
at = "at " + g.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream)
|
||||
}
|
||||
|
||||
// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard.
|
||||
const keptGivenUp = 1024
|
||||
|
||||
// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use.
|
||||
type DeadLetters struct {
|
||||
mu sync.Mutex
|
||||
heard map[string]GivenUp
|
||||
order []string
|
||||
watchers map[string][]chan GivenUp
|
||||
}
|
||||
|
||||
// Heard takes one given-up ask in.
|
||||
func (d *DeadLetters) Heard(g GivenUp) {
|
||||
d.mu.Lock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if _, ok := d.heard[g.AskID]; !ok {
|
||||
d.order = append(d.order, g.AskID)
|
||||
if len(d.order) > keptGivenUp {
|
||||
delete(d.heard, d.order[0])
|
||||
d.order = d.order[1:]
|
||||
}
|
||||
}
|
||||
d.heard[g.AskID] = g
|
||||
ws := d.watchers[g.AskID]
|
||||
delete(d.watchers, g.AskID)
|
||||
d.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- g // buffered, one each
|
||||
}
|
||||
}
|
||||
|
||||
// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel.
|
||||
func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) {
|
||||
ch := make(chan GivenUp, 1)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if g, ok := d.heard[askID]; ok {
|
||||
ch <- g
|
||||
return ch, func() {}
|
||||
}
|
||||
d.watchers[askID] = append(d.watchers[askID], ch)
|
||||
return ch, func() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
ws := d.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == ch {
|
||||
d.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(d.watchers[askID]) == 0 {
|
||||
delete(d.watchers, askID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type notTakenKey struct{}
|
||||
|
||||
// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's).
|
||||
func NotTakenBound(ctx context.Context, bound time.Duration) context.Context {
|
||||
return context.WithValue(ctx, notTakenKey{}, bound)
|
||||
}
|
||||
|
||||
func notTakenBoundOf(ctx context.Context) time.Duration {
|
||||
d, _ := ctx.Value(notTakenKey{}).(time.Duration)
|
||||
return d
|
||||
}
|
||||
|
||||
// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under
|
||||
// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on
|
||||
// the node "bus". Every step the build module writes is handed on as it is.
|
||||
type LostAskWaiter struct {
|
||||
Inner AskWaiter
|
||||
Dead *DeadLetters
|
||||
// Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is
|
||||
// the event that makes hearing matter.
|
||||
Hearing interface {
|
||||
Deaf() string
|
||||
Again()
|
||||
}
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (w LostAskWaiter) now() time.Time {
|
||||
if w.Now != nil {
|
||||
return w.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's.
|
||||
func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if w.Hearing != nil && w.Hearing.Deaf() != "" {
|
||||
w.Hearing.Again()
|
||||
}
|
||||
inner, err := w.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dead <-chan GivenUp
|
||||
cancel := func() {}
|
||||
if w.Dead != nil {
|
||||
dead, cancel = w.Dead.watch(askID)
|
||||
}
|
||||
// The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask.
|
||||
var notTaken <-chan struct{}
|
||||
bound := notTakenBoundOf(ctx)
|
||||
stopBound := func() {}
|
||||
if bound > 0 {
|
||||
var nt context.Context
|
||||
nt, stopBound = context.WithTimeout(context.Background(), bound)
|
||||
notTaken = nt.Done()
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer cancel()
|
||||
defer func() { stopBound() }()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
ended := func(reason string) delivery.Step {
|
||||
at := w.now().UTC()
|
||||
return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-inner:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if notTaken != nil {
|
||||
stopBound() // a node took it: its step is bounded by the asker from its start
|
||||
notTaken = nil
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case g := <-dead:
|
||||
dead = nil
|
||||
send(ended(g.Words()))
|
||||
return
|
||||
case <-notTaken:
|
||||
send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound)))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords).
|
||||
func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) }
|
||||
|
||||
// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound.
|
||||
func IsNotTaken(s delivery.Step) bool {
|
||||
return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords)
|
||||
}
|
||||
|
||||
// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is
|
||||
// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is
|
||||
// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in
|
||||
// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every
|
||||
// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so
|
||||
// the module's health tool says it: a deaf asker is not healthy.
|
||||
|
||||
// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not.
|
||||
type Hearing struct {
|
||||
Heard bool `json:"heard"`
|
||||
Streams []string `json:"streams"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing).
|
||||
const HearingKey = "dead-letters"
|
||||
|
||||
// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to
|
||||
// heard; it answers the stop, or why it cannot.
|
||||
type DeadLetterFollower interface {
|
||||
Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error)
|
||||
}
|
||||
|
||||
// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the
|
||||
// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again.
|
||||
type JetStreamDeadLetters struct{ JS jetstream.JetStream }
|
||||
|
||||
// Follow is DeadLetterFollower's.
|
||||
func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) {
|
||||
cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
if g, ok := GivenUpOf(m.Headers()); ok {
|
||||
heard(g)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cc.Stop, nil
|
||||
}
|
||||
|
||||
// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing.
|
||||
type DeadLetterHearing struct {
|
||||
follower DeadLetterFollower
|
||||
streams []string
|
||||
dead *DeadLetters
|
||||
logf func(string, ...any)
|
||||
said func(Hearing)
|
||||
ctx context.Context
|
||||
since time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
stops map[string]func()
|
||||
why map[string]string
|
||||
trying bool
|
||||
again bool
|
||||
told *Hearing
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said
|
||||
// is told each change of whether it hears (nil tells nothing); ctx ends it.
|
||||
func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters,
|
||||
logf func(string, ...any), said func(Hearing)) *DeadLetterHearing {
|
||||
h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(),
|
||||
stops: map[string]func(){}, why: map[string]string{}}
|
||||
for _, l := range languages {
|
||||
s := SeatStream(l)
|
||||
h.streams = append(h.streams, s)
|
||||
h.why[s] = "not tried yet"
|
||||
}
|
||||
if h.logf == nil {
|
||||
h.logf = func(string, ...any) {}
|
||||
}
|
||||
h.Again()
|
||||
return h
|
||||
}
|
||||
|
||||
// Deaf says why a stream is not heard, or "" when every one is.
|
||||
func (h *DeadLetterHearing) Deaf() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.deafLocked()
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) deafLocked() string {
|
||||
var out []string
|
||||
for _, s := range h.streams {
|
||||
if why, deaf := h.why[s]; deaf {
|
||||
out = append(out, s+": "+why)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " +
|
||||
strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after.
|
||||
func (h *DeadLetterHearing) Again() {
|
||||
h.mu.Lock()
|
||||
if h.trying {
|
||||
h.again = true
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.trying = true
|
||||
h.mu.Unlock()
|
||||
h.wg.Add(1)
|
||||
go func() {
|
||||
defer h.wg.Done()
|
||||
for {
|
||||
h.try()
|
||||
h.mu.Lock()
|
||||
if !h.again || h.ctx.Err() != nil {
|
||||
h.trying = false
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.again = false
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait returns when no try runs.
|
||||
func (h *DeadLetterHearing) Wait() { h.wg.Wait() }
|
||||
|
||||
// Stop ends every stream's following.
|
||||
func (h *DeadLetterHearing) Stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for s, stop := range h.stops {
|
||||
stop()
|
||||
delete(h.stops, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) try() {
|
||||
for _, s := range h.streams {
|
||||
h.mu.Lock()
|
||||
_, deaf := h.why[s]
|
||||
h.mu.Unlock()
|
||||
if !deaf || h.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
stream := s
|
||||
stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) {
|
||||
h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream,
|
||||
g.Deliveries, g.At.Format(time.RFC3339))
|
||||
h.dead.Heard(g)
|
||||
})
|
||||
h.mu.Lock()
|
||||
if err != nil {
|
||||
h.why[stream] = err.Error()
|
||||
h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v",
|
||||
stream, DeadLetterFilter(stream), DeadLettersStream, err)
|
||||
} else {
|
||||
delete(h.why, stream)
|
||||
h.stops[stream] = stop
|
||||
h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
h.tell()
|
||||
}
|
||||
|
||||
// tell says the hearing when it changed.
|
||||
func (h *DeadLetterHearing) tell() {
|
||||
h.mu.Lock()
|
||||
why := h.deafLocked()
|
||||
now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()}
|
||||
same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why
|
||||
if !same {
|
||||
h.told = &now
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !same && h.said != nil {
|
||||
h.said(now)
|
||||
}
|
||||
}
|
||||
|
||||
// HearingState is the name of the state an asker keeps its Hearing in (`<module>_hearing`, key HearingKey), which its
|
||||
// health tool reads.
|
||||
const HearingState = "hearing"
|
||||
|
||||
// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be
|
||||
// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet.
|
||||
type HearingKeeper interface {
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and
|
||||
// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits.
|
||||
func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) {
|
||||
if kv == nil {
|
||||
if logf != nil {
|
||||
logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return func(h Hearing) {
|
||||
data, err := json.Marshal(h)
|
||||
if err == nil {
|
||||
_, err = kv.Put(ctx, HearingKey, data)
|
||||
}
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil {
|
||||
logf("whether %s is heard could not be kept: %v", DeadLettersStream, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears
|
||||
// DEAD_LETTERS, unhealthy saying why while it does not or has not said.
|
||||
func HearingHealth(raw []byte) health.Answer {
|
||||
if len(raw) == 0 {
|
||||
return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there")
|
||||
}
|
||||
var h Hearing
|
||||
if err := json.Unmarshal(raw, &h); err != nil {
|
||||
return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error())
|
||||
}
|
||||
if !h.Heard {
|
||||
return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)")
|
||||
}
|
||||
return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", "))
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package buildrun
|
||||
|
||||
// The build modules the mesh runs, by the language each builds (ADR 0469).
|
||||
const (
|
||||
GoBuildModule = "mesh-build-go"
|
||||
NodeBuildModule = "mesh-build-node"
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
|
||||
const DefaultCapacity = 1
|
||||
|
||||
// Places counts the build steps a part runs on each node and holds a lane back while its node is at its
|
||||
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
|
||||
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
|
||||
// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's
|
||||
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
|
||||
// other lanes, not by a timer: ending a lane wakes the waiting ones.
|
||||
type Places struct {
|
||||
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
|
||||
Capacity map[string]int
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]int // lanes holding a place
|
||||
reserved map[string]int // lanes chosen for the node that do not hold a place yet
|
||||
changed chan struct{} // closed, and replaced, each time a lane ends
|
||||
}
|
||||
|
||||
// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not
|
||||
// name, and 0 for one it names so, where the part never builds.
|
||||
func (p *Places) CapacityOf(node string) int {
|
||||
if n, ok := p.Capacity[node]; ok && n >= 0 {
|
||||
return n
|
||||
}
|
||||
return DefaultCapacity
|
||||
}
|
||||
|
||||
// Running is the steps in flight now, by node.
|
||||
func (p *Places) Running() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.running))
|
||||
for n, c := range p.running {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Reserved is the lanes chosen for a node that hold no place yet, by node.
|
||||
func (p *Places) Reserved() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.reserved))
|
||||
for n, c := range p.reserved {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Places) init() {
|
||||
if p.running == nil {
|
||||
p.running = map[string]int{}
|
||||
}
|
||||
if p.reserved == nil {
|
||||
p.reserved = map[string]int{}
|
||||
}
|
||||
if p.changed == nil {
|
||||
p.changed = make(chan struct{})
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
|
||||
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
|
||||
// gives up the reservation too.
|
||||
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
|
||||
for {
|
||||
p.mu.Lock()
|
||||
p.init()
|
||||
if p.running[node] < p.CapacityOf(node) {
|
||||
p.running[node]++
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
p.mu.Lock()
|
||||
p.running[node]--
|
||||
close(p.changed)
|
||||
p.changed = make(chan struct{})
|
||||
p.mu.Unlock()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
wait := p.changed
|
||||
p.mu.Unlock()
|
||||
select {
|
||||
case <-wait:
|
||||
case <-ctx.Done():
|
||||
p.Cancel(node)
|
||||
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel gives up a place reserved for the node that will not be taken.
|
||||
func (p *Places) Cancel(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
}
|
||||
|
||||
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
|
||||
// them, then by name), keeps the first want and reserves a place on each, all at once.
|
||||
func (p *Places) choose(nodes []string, want int) []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
load := func(n string) int { return p.running[n] + p.reserved[n] }
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
|
||||
if ri != rj {
|
||||
return ri
|
||||
}
|
||||
if load(nodes[i]) != load(nodes[j]) {
|
||||
return load(nodes[i]) < load(nodes[j])
|
||||
}
|
||||
return nodes[i] < nodes[j]
|
||||
})
|
||||
if len(nodes) > want {
|
||||
nodes = nodes[:want]
|
||||
}
|
||||
for _, n := range nodes {
|
||||
p.reserved[n]++
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func acquireAsync(p *Places, node string) chan func() {
|
||||
got := make(chan func(), 1)
|
||||
go func() {
|
||||
release, err := p.Acquire(context.Background(), node)
|
||||
if err != nil {
|
||||
close(got)
|
||||
return
|
||||
}
|
||||
got <- release
|
||||
}()
|
||||
return got
|
||||
}
|
||||
|
||||
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"ace": 2}}
|
||||
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
|
||||
ra, rb := <-a, <-b
|
||||
third := acquireAsync(p, "ace")
|
||||
select {
|
||||
case <-third:
|
||||
t.Fatal("a third step started on a node of capacity 2 with two running")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if p.Running()["ace"] != 2 {
|
||||
t.Errorf("running = %v, want 2 on ace", p.Running())
|
||||
}
|
||||
ra()
|
||||
select {
|
||||
case r := <-third:
|
||||
r()
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the third step did not start when one ended")
|
||||
}
|
||||
rb()
|
||||
}
|
||||
|
||||
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3}}
|
||||
first := <-acquireAsync(p, "g14")
|
||||
second := acquireAsync(p, "g14")
|
||||
select {
|
||||
case <-second:
|
||||
t.Fatal("two steps on a node without a setting")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
first()
|
||||
r := <-second
|
||||
r()
|
||||
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
|
||||
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
|
||||
p := &Places{}
|
||||
r := <-acquireAsync(p, "ace")
|
||||
defer r()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Error("Acquire returned no error for a cancelled wait")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a cancelled wait did not return")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
|
||||
func TestACapacityOfZeroKeepsThePartOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3, "ace": 2, "g14": 0, "novox": 0}}
|
||||
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
|
||||
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
|
||||
}
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
|
||||
for i := 0; i < 6; i++ {
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n == "g14" || n == "novox" {
|
||||
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
|
||||
_, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "capacity") {
|
||||
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
|
||||
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
|
||||
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
|
||||
type PresenceSet struct {
|
||||
mu sync.Mutex
|
||||
online map[string]bool
|
||||
}
|
||||
|
||||
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
|
||||
// value that is not a presence are left out.
|
||||
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, part, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || part != "" {
|
||||
return false
|
||||
}
|
||||
on := false
|
||||
if !deleted {
|
||||
p, err := delivery.DecodePresence(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
on = p.Online
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.online[node] == on {
|
||||
return false
|
||||
}
|
||||
if s.online == nil {
|
||||
s.online = map[string]bool{}
|
||||
}
|
||||
if on {
|
||||
s.online[node] = true
|
||||
} else {
|
||||
delete(s.online, node)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Online is whether the watchman last saw the node online.
|
||||
func (s *PresenceSet) Online(node string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.online[node]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func presenceValue(t *testing.T, online bool) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
|
||||
var s PresenceSet
|
||||
if s.Online("ace") {
|
||||
t.Error("a node the watchman never wrote is online")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
|
||||
t.Error("ace online was not taken in")
|
||||
}
|
||||
if s.Put("ace", presenceValue(t, true), false) {
|
||||
t.Error("the same presence again counted as a change")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
|
||||
t.Error("ace going offline was not taken in")
|
||||
}
|
||||
s.Put("g14", presenceValue(t, true), false)
|
||||
if !s.Put("g14", nil, true) || s.Online("g14") {
|
||||
t.Error("a deleted key is still online")
|
||||
}
|
||||
// A part's instance (part.<part>.<instance>) is not a node.
|
||||
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
|
||||
t.Error("a part's key was taken for a node")
|
||||
}
|
||||
if s.Put("novox", []byte("junk"), false) {
|
||||
t.Error("junk counted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the
|
||||
// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left
|
||||
// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// ToolchainSet): an entry is passed on only while the placement (the seat mesh-placement's state `placement`, held by
|
||||
// the assigner, delivery.PlacementBucket, read by delivery.Placed; ADR 0567, ADR 0549 §3) places its build module on
|
||||
// its node, and only once it was written since that placement; it is passed on as deleted otherwise, said once, and
|
||||
// passed on when its node writes it again. A report written before the placement is one an earlier run of the module
|
||||
// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23
|
||||
// back into the agreement before the module ran there). Until the placement is settled, and while no node's placement
|
||||
// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed
|
||||
// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465,
|
||||
// and it stays.
|
||||
//
|
||||
// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's
|
||||
// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6).
|
||||
type ToolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
placed *delivery.Placed
|
||||
known bool // the placement is settled
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what
|
||||
// it leaves out.
|
||||
func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate {
|
||||
g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: delivery.NewPlaced(),
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Placement takes one revision of the seat mesh-placement's placement in (at is when the bus wrote it) and answers whether what it
|
||||
// passed on changed. A value that does not read is said, and places nothing on its node.
|
||||
func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was := g.placed.Seeded()
|
||||
node, changed, err := g.placed.Put(key, value, deleted, at)
|
||||
if err != nil {
|
||||
g.logf("%s: %v (issue 1940)", delivery.PlacementBucket, err)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
if was != g.placed.Seeded() {
|
||||
// The first placement came, or the last went: every report's standing changed.
|
||||
return g.all()
|
||||
}
|
||||
return g.again(node, changed)
|
||||
}
|
||||
|
||||
// Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's
|
||||
// placement is held. It answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
return g.all()
|
||||
}
|
||||
|
||||
// all passes on every report again. The caller holds mu.
|
||||
func (g *ToolchainGate) all() bool {
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// again passes on every report of node by a build module among modules. The caller holds mu.
|
||||
func (g *ToolchainGate) again(node string, modules []string) bool {
|
||||
changed := false
|
||||
for _, module := range modules {
|
||||
if !g.modules[module] {
|
||||
continue
|
||||
}
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the placement is not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *ToolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known && g.placed.Seeded() {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its
|
||||
// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *ToolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, placed := g.placed.Since(node, module)
|
||||
if !placed {
|
||||
return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FollowPlacement reads the seat mesh-placement's placement with its history into gate, says it settled, and follows it; changed
|
||||
// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part
|
||||
// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that
|
||||
// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel.
|
||||
func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, delivery.PlacementBucket)
|
||||
if err != nil {
|
||||
logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
put := func(e jetstream.KeyValueEntry) bool {
|
||||
return gate.Placement(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut, e.Created())
|
||||
}
|
||||
for settled := false; !settled; {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = w.Stop()
|
||||
return errs, ctx.Err()
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errs, errors.New("the watch of " + delivery.PlacementBucket + " ended before it settled")
|
||||
}
|
||||
if e == nil {
|
||||
settled = true
|
||||
continue
|
||||
}
|
||||
put(e)
|
||||
}
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs <- ctx.Err()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
errs <- errors.New("the watch of " + delivery.PlacementBucket + " ended")
|
||||
return
|
||||
}
|
||||
if e != nil && put(e) {
|
||||
changed(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
// placing is a value of the state `placement` placing modules on node.
|
||||
func placing(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
return key, values[key]
|
||||
}
|
||||
|
||||
// placingLegacy is placing in the form before, `placement.<node>` (research 072 row 37).
|
||||
func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
key, _ := delivery.LegacyPlacementKey(node)
|
||||
value, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while
|
||||
// the placement places its build module on its node, and only once written since; before the placement is settled, and
|
||||
// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left
|
||||
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
|
||||
// back before the module ran there.
|
||||
func TestToolchainGate(t *testing.T) {
|
||||
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
|
||||
three := []string{"g14", "novox", "shanks"}
|
||||
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
|
||||
set := &ToolchainSet{}
|
||||
said := &[]string{}
|
||||
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range three {
|
||||
if seeded {
|
||||
key, value := placing(t, node, NodeBuildModule)
|
||||
gate.Placement(key, value, false, old.Add(-time.Hour))
|
||||
}
|
||||
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
}
|
||||
return gate, set, said
|
||||
}
|
||||
|
||||
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
|
||||
t.Fatalf("before the placement is settled every report counts; got %+v", a)
|
||||
}
|
||||
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
|
||||
gate.Placement(k, v, false, old)
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
|
||||
t.Fatalf("ace's report is said left out once; said %q", *said)
|
||||
}
|
||||
if gate.Toolchain(NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node not placed changed what was passed on")
|
||||
}
|
||||
// Placed on ace with a report written since: it counts, and the builders really disagree.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
gate.Placement(k, v, false, old.Add(-time.Minute))
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
|
||||
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
|
||||
}
|
||||
// Taken off ace again (its key deleted): its report goes again.
|
||||
if !gate.Placement(k, nil, true, placed) {
|
||||
t.Fatal("taking ace's placement away passed nothing on")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is placed or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1", placed)
|
||||
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
|
||||
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
|
||||
}
|
||||
})
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"a report left before the placement", old, false},
|
||||
{"a report written since the placement", placed.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
k, v := placing(t, "ace") // ace's key, first heard placing nothing
|
||||
gate.Placement(k, v, false, old.Add(-time.Hour))
|
||||
gate.Settled()
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed) {
|
||||
t.Fatal("a placement with no report passed something on")
|
||||
}
|
||||
key, value := gateReport(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// ace's placement written again with another module beside: its standing does not move, nothing is said again.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed.Add(time.Minute)) {
|
||||
t.Fatal("the same placement of mesh-build-node changed what was passed on")
|
||||
}
|
||||
if c.counts {
|
||||
return
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", *said)
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
|
||||
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("no placement held: every report counts", func(t *testing.T) {
|
||||
gate, set, _ := start(t, false)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
gate.Settled()
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("before the first placement file every report counts; got %d", n)
|
||||
}
|
||||
k, v := placing(t, "g14", NodeBuildModule)
|
||||
if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
|
||||
t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript")))
|
||||
}
|
||||
if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("the last placement gone: every report counts again")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
|
||||
p := delivery.NewPlaced()
|
||||
// ace in the form before, then converted: its module stands since before, the legacy key read no more.
|
||||
k, v := placingLegacy(t, "ace", NodeBuildModule)
|
||||
p.Put(k, v, false, old)
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil {
|
||||
t.Errorf("the conversion of ace's key changed %v, %v", changed, err)
|
||||
}
|
||||
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
|
||||
t.Errorf("a module key says which modules stand: %v", changed)
|
||||
}
|
||||
if k, v := placingLegacy(t, "ace"); true {
|
||||
if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 {
|
||||
t.Errorf("the legacy key is read after the node key: %v", changed)
|
||||
}
|
||||
}
|
||||
k2, v2 := placing(t, "g14")
|
||||
p.Put(k2, v2, false, old)
|
||||
k2, v2 = placing(t, "g14", NodeBuildModule)
|
||||
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
|
||||
t.Errorf("a module placed says changed %v", changed)
|
||||
}
|
||||
if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil {
|
||||
t.Error("a value that does not read was not said")
|
||||
}
|
||||
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
|
||||
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
|
||||
}
|
||||
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
|
||||
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
|
||||
}
|
||||
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
|
||||
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement")
|
||||
gate, _, _ := start(t, false)
|
||||
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
|
||||
if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
|
||||
t.Errorf("a refused placement stopped the start: %v", err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
|
||||
type Caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
|
||||
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
|
||||
// ends or ctx does.
|
||||
type Steps interface {
|
||||
Wait(ctx context.Context, module, key string) (delivery.Step, error)
|
||||
}
|
||||
|
||||
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
|
||||
// key when the step ends (ADR 0469 clause 2).
|
||||
type Runner struct {
|
||||
Caller Caller
|
||||
Steps Steps
|
||||
// Bound is how long a step may run before the builder stops waiting for it.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
|
||||
// failed or was refused is an error naming the node, the state and the reason.
|
||||
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
|
||||
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
var answer struct {
|
||||
Step string `json:"step"`
|
||||
Key string `json:"key"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
|
||||
}
|
||||
if answer.Key == "" {
|
||||
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
|
||||
}
|
||||
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
|
||||
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
|
||||
}
|
||||
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
|
||||
defer cancel()
|
||||
step, err := r.Steps.Wait(wait, module, answer.Key)
|
||||
if err != nil {
|
||||
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
|
||||
err = cause
|
||||
}
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
|
||||
}
|
||||
return step.Result, nil
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
type call struct {
|
||||
node, key string
|
||||
body any
|
||||
}
|
||||
|
||||
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
|
||||
type fakeCaller struct {
|
||||
calls []call
|
||||
answer string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
f.calls = append(f.calls, call{node, key, body})
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return json.RawMessage(f.answer), nil
|
||||
}
|
||||
|
||||
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
|
||||
type fakeSteps struct {
|
||||
mu sync.Mutex
|
||||
steps map[string]delivery.Step
|
||||
// during, when set, is called as a wait starts and the function it answers as the wait ends.
|
||||
during func(key string) func()
|
||||
}
|
||||
|
||||
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
s, ok := f.steps[module+"/"+key]
|
||||
during := f.during
|
||||
f.mu.Unlock()
|
||||
if ok {
|
||||
if during != nil {
|
||||
defer during(key)()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return delivery.Step{}, context.Cause(ctx)
|
||||
}
|
||||
|
||||
func stepAt(node, id, state, reason string, result string) delivery.Step {
|
||||
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
|
||||
if result != "" {
|
||||
s.Result = json.RawMessage(result)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func runCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
|
||||
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
|
||||
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
|
||||
}
|
||||
var fr struct{ Workspace string }
|
||||
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
|
||||
t.Errorf("result = %s (%v), want the step's result", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func runFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
|
||||
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
|
||||
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
|
||||
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
|
||||
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
|
||||
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if err == nil {
|
||||
t.Errorf("answer %s: no error", answer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
|
||||
boom := errors.New("no responders")
|
||||
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("error = %v, want the call's error naming the node", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A step run on its node and read from its steps key, one table of cases (the test budget).
|
||||
func TestRunningAStepOnANode(t *testing.T) {
|
||||
t.Run("run calls the tool on the node and reads the steps key", runCallsTheToolOnTheNodeAndReadsTheStepsKey)
|
||||
t.Run("run fails on a failed or refused step naming its reason", runFailsOnAFailedOrRefusedStepNamingItsReason)
|
||||
t.Run("run stops waiting at the bound", runStopsWaitingAtTheBound)
|
||||
t.Run("run refuses an answer with no key and a key of another node", runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode)
|
||||
t.Run("run returns the calls own error", runReturnsTheCallsOwnError)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
|
||||
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
|
||||
|
||||
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
|
||||
// put them in. Safe for use from the watches and from a build at once.
|
||||
type ToolchainSet struct {
|
||||
mu sync.Mutex
|
||||
by map[string]map[string]delivery.Toolchain // language -> node -> entry
|
||||
}
|
||||
|
||||
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
|
||||
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
|
||||
// A key or a value that is not an entry of the module's language is left out.
|
||||
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
|
||||
language, ok := LanguageOf[module]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
node, keyLanguage, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || keyLanguage != language {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := s.by[language][node]; !had {
|
||||
return false
|
||||
}
|
||||
delete(s.by[language], node)
|
||||
return true
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err != nil || tc.Language != language {
|
||||
return false
|
||||
}
|
||||
if s.by == nil {
|
||||
s.by = map[string]map[string]delivery.Toolchain{}
|
||||
}
|
||||
if s.by[language] == nil {
|
||||
s.by[language] = map[string]delivery.Toolchain{}
|
||||
}
|
||||
cur, had := s.by[language][node]
|
||||
s.by[language][node] = tc
|
||||
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
|
||||
}
|
||||
|
||||
// Entries are the nodes' entries of a language, in no particular order.
|
||||
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []delivery.NodeToolchain
|
||||
for node, tc := range s.by[language] {
|
||||
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func tcValue(t *testing.T, language, version string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
|
||||
var s ToolchainSet
|
||||
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("a first entry did not count as a change")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("the same entry again counted as a change")
|
||||
}
|
||||
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
|
||||
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
if got := s.Entries("go"); len(got) != 2 {
|
||||
t.Errorf("go entries = %+v, want two", got)
|
||||
}
|
||||
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
|
||||
t.Errorf("typescript entries = %+v", got)
|
||||
}
|
||||
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
|
||||
t.Error("a delete did not remove the node's entry")
|
||||
}
|
||||
// What is not a toolchain of the module's language is left out.
|
||||
s = ToolchainSet{}
|
||||
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
|
||||
t.Error("a typescript toolchain in the go build module's bucket counted")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
|
||||
t.Error("garbage or an unknown module counted")
|
||||
}
|
||||
if len(s.Entries("go")) != 0 {
|
||||
t.Error("an entry was kept")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
||||
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
||||
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
||||
// other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
{Name: build.AskAgainTool,
|
||||
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
||||
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
||||
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
||||
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := askAgain(key, value); err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
||||
}
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.BuildAgainTool,
|
||||
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
||||
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
||||
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
||||
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
||||
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
||||
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Revision == 0 && !r.Pending {
|
||||
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
||||
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
||||
}},
|
||||
{Name: delivery.BuildsSinceTool,
|
||||
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
|
||||
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
|
||||
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
|
||||
"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
|
||||
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
|
||||
}
|
||||
return build.BuildsSince(since, at)
|
||||
}},
|
||||
{Name: build.AnnounceAgainTool,
|
||||
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
|
||||
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
|
||||
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
|
||||
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
|
||||
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused || r.Revision == 0 {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
||||
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
||||
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
||||
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
||||
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
language := build.LanguageOf(str(args, "language"))
|
||||
if language == "" {
|
||||
language = build.LanguageOf(str(args, "seat"))
|
||||
}
|
||||
if language == "" {
|
||||
language = delivery.AskGo
|
||||
}
|
||||
limit, _ := args["limit"].(float64)
|
||||
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return queue(key, value)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
||||
a, err := build.DecodeQueueAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
}
|
||||
|
||||
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
||||
// bound for a value written after it that answered says is the builder's answer.
|
||||
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
heard := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if answered(c.Value) {
|
||||
select {
|
||||
case heard <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
||||
return e.Value, nil
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-heard:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) build.StateRead {
|
||||
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return e.Value, time.Time{}, nil
|
||||
}, Written: func(key string) time.Time {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return written(s, key, e.Revision)
|
||||
}}
|
||||
}
|
||||
|
||||
// written is when the key's revision was written, from its history, or zero.
|
||||
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||
h, err := s.History(key)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, r := range h {
|
||||
if r.Revision == revision {
|
||||
return r.At
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func main() {
|
||||
steps := map[string]build.StateRead{}
|
||||
for _, bm := range build.StuckBuildModules {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
||||
a, err := build.DecodeReAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,11 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return settled(ctx, w, each)
|
||||
}
|
||||
|
||||
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
|
||||
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -118,5 +123,6 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
|
||||
return catalogue.Decode([]byte(kept.JSON))
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||
// controller had never taken in, and the builder kept them done.
|
||||
|
||||
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
|
||||
// answer}}; a reference is gone while gone says so.
|
||||
type heldVerb struct {
|
||||
mu sync.Mutex
|
||||
@@ -29,7 +30,7 @@ type heldVerb struct {
|
||||
|
||||
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||
t.Helper()
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
References []string `json:"references"`
|
||||
}
|
||||
|
||||
@@ -4,73 +4,303 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
||||
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||
// be answered (issue 1945). Never polled.
|
||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||
|
||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
seen, err := h.seen(ctx, a, false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
return seen.Step || seen.OnStream, nil
|
||||
}
|
||||
|
||||
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
|
||||
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
|
||||
seen, err := h.seen(ctx, a, true)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return "", err
|
||||
}
|
||||
return seen.Why(), nil
|
||||
}
|
||||
|
||||
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
|
||||
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
|
||||
const (
|
||||
maxOnSubject = 5000
|
||||
maxSteps = 200
|
||||
)
|
||||
|
||||
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
|
||||
// the ask is on its stream.
|
||||
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
|
||||
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return seen, err
|
||||
}
|
||||
for range keys.Keys() {
|
||||
keys.Stop()
|
||||
return true, nil // a node took it
|
||||
seen.Step = true // a node took it
|
||||
return seen, nil
|
||||
}
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
seen.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return seen, nil
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return seen, err
|
||||
}
|
||||
var ahead []build.Queued
|
||||
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seen.OnStream, seen.PutAt = true, q.PutAt
|
||||
return false
|
||||
}
|
||||
ahead = append(ahead, q)
|
||||
return true
|
||||
})
|
||||
if err != nil || !seen.OnStream || !later {
|
||||
return seen, err
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen.Ahead = ahead
|
||||
since := seen.At
|
||||
if since.IsZero() {
|
||||
since = seen.PutAt
|
||||
}
|
||||
for _, q := range ahead {
|
||||
if q.At.Before(since) {
|
||||
since = q.At
|
||||
}
|
||||
}
|
||||
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
// stream is the build seat's stream of language, its name and the subject of kind on it.
|
||||
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
|
||||
subject, err := delivery.AskSubject(language, kind)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
}
|
||||
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||
// messages by sequence and nothing wider (issue 1910).
|
||||
name, err := delivery.AskStream(a.Language)
|
||||
name, err := delivery.AskStream(language)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return nil, "", "", err
|
||||
}
|
||||
stream, err := h.js.Stream(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
||||
// instead, each a direct read that consumes nothing.
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
||||
m, err := stream.GetMsg(ctx, seq)
|
||||
return stream, name, subject, err
|
||||
}
|
||||
|
||||
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
|
||||
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
|
||||
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
|
||||
// none consumed.
|
||||
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
|
||||
for seq, n := uint64(1), 0; ; n++ {
|
||||
if n >= maxOnSubject {
|
||||
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
|
||||
}
|
||||
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
continue // taken and acknowledged, or removed
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
||||
return true, nil
|
||||
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
|
||||
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
|
||||
q.Kind = kind
|
||||
}
|
||||
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
|
||||
q.At = q.PutAt
|
||||
}
|
||||
if !each(q) {
|
||||
return nil
|
||||
}
|
||||
seq = m.Sequence + 1
|
||||
}
|
||||
}
|
||||
|
||||
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
|
||||
// ask id and is left out).
|
||||
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
|
||||
all, err := steps.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return map[string][]string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer all.Stop()
|
||||
out := map[string][]string{}
|
||||
for k := range all.Keys() {
|
||||
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
|
||||
out[id] = append(out[id], k)
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stepState is the state of the first of keys a node wrote, "" for none.
|
||||
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
|
||||
for _, k := range keys {
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if s, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
return s.State
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
|
||||
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
|
||||
// (mesh-sdk buildstep).
|
||||
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
|
||||
var ids []string
|
||||
for id := range taken {
|
||||
if build.AskedAt(id).After(since) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
|
||||
read := 0
|
||||
for _, id := range ids {
|
||||
for _, k := range taken[id] {
|
||||
if read >= maxSteps {
|
||||
return nil
|
||||
}
|
||||
read++
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ok || !build.Proves(stepKind, kind) {
|
||||
continue
|
||||
}
|
||||
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
|
||||
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
|
||||
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var seq uint64
|
||||
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seq = q.Seq
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); err != nil || seq == 0 {
|
||||
return err
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
|
||||
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||
module, err := delivery.BuildModuleOf(language)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
|
||||
q.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
q.Note = name + " does not exist"
|
||||
return q, nil
|
||||
}
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
var ahead []build.Queued
|
||||
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
|
||||
ahead = append(ahead, m)
|
||||
return len(ahead) < limit
|
||||
}); err != nil {
|
||||
return q, err
|
||||
}
|
||||
if len(ahead) == 0 {
|
||||
continue
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
|
||||
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
@@ -22,8 +22,15 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
// The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1),
|
||||
// another repository's, by its published version, so the builder is tested against the real pulling code.
|
||||
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -49,7 +56,7 @@ const (
|
||||
)
|
||||
|
||||
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
|
||||
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// its build seat's asks on its nodes (runner.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
|
||||
type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
@@ -64,14 +71,14 @@ type fakeBuildModules struct {
|
||||
// closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks
|
||||
// waiting in their streams).
|
||||
closed atomic.Bool
|
||||
pullers []*buildstep.Puller
|
||||
pullers []*runner.Puller
|
||||
}
|
||||
|
||||
// open lets the nodes pull again and wakes their pullers.
|
||||
func (f *fakeBuildModules) open() {
|
||||
f.closed.Store(false)
|
||||
f.mu.Lock()
|
||||
ps := append([]*buildstep.Puller(nil), f.pullers...)
|
||||
ps := append([]*runner.Puller(nil), f.pullers...)
|
||||
f.mu.Unlock()
|
||||
for _, p := range ps {
|
||||
p.Wake()
|
||||
@@ -135,26 +142,26 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
language := strings.TrimPrefix(module, "mesh-build-")
|
||||
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
return kv.Put(f.ctx, key, raw)
|
||||
}
|
||||
call := func(name string) buildstep.StepFunc {
|
||||
call := func(name string) runner.StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
|
||||
}
|
||||
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
steps := runner.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
for _, node := range nodes {
|
||||
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
p := &runner.Puller{Source: &runner.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
Capacity: func() int {
|
||||
if f.closed.Load() {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }}
|
||||
f.mu.Lock()
|
||||
f.pullers = append(f.pullers, p)
|
||||
f.mu.Unlock()
|
||||
@@ -162,11 +169,29 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||
const trunkHead = "3333333333333333333333333333333333333333"
|
||||
|
||||
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
candidates jetstream.KeyValue
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
instances jetstream.KeyValue
|
||||
@@ -243,9 +268,16 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
}
|
||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
|
||||
logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"},
|
||||
AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||
b.fake.closed.Store(closed)
|
||||
b.fake.serve(t, "mesh-build-go", "n1", "n2")
|
||||
@@ -258,6 +290,29 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
return b
|
||||
}
|
||||
|
||||
// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user.
|
||||
func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry {
|
||||
t.Helper()
|
||||
conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []delivery.BuildLogEntry
|
||||
for _, le := range entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(le.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
||||
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
||||
t.Helper()
|
||||
@@ -324,7 +379,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -384,6 +439,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
|
||||
t.Errorf("toolchains = %+v", v.Toolchains)
|
||||
}
|
||||
// 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a
|
||||
// reader with the log's read reads them, the end naming the manifest and what was made.
|
||||
ended := b.logged(t, "netcheck")
|
||||
if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent ||
|
||||
ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" {
|
||||
t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended)
|
||||
}
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
|
||||
if got := b.fake.count(tool); got != want {
|
||||
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
|
||||
@@ -408,15 +470,20 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4).
|
||||
func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "."
|
||||
// is given to the build tools as it is (1615/AC4). One table: the two tests were merged to pay for the enrolment's.
|
||||
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
for _, c := range []struct{ name, path string }{{"no toolchain", "modules/ca-trust"}, {"root", delivery.RootPath}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", c.path, commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -471,14 +538,6 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
|
||||
}
|
||||
}
|
||||
|
||||
// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
|
||||
func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", delivery.RootPath, commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
}
|
||||
|
||||
func mustKey(t *testing.T, module string) string {
|
||||
k, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
@@ -513,7 +572,9 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
|
||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||
// fingerprint first appeared at).
|
||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
@@ -530,7 +591,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
||||
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -550,12 +611,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
defer b.fake.mu.Unlock()
|
||||
found := false
|
||||
for _, f := range b.fake.fetches {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -603,7 +664,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
||||
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -801,6 +862,34 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||
}
|
||||
}
|
||||
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||
held := heldOnTheBus{b.js}
|
||||
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
|
||||
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued := func() []build.QueuedState {
|
||||
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
|
||||
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
|
||||
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
|
||||
}
|
||||
return q.Kinds[0].Messages
|
||||
}
|
||||
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
|
||||
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
|
||||
}
|
||||
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
|
||||
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
|
||||
}
|
||||
if err := held.Drop(b.ctx, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
|
||||
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
|
||||
}
|
||||
b.stop()
|
||||
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||
b.fake.open()
|
||||
|
||||
@@ -6,22 +6,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -47,103 +45,32 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
cred, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
conn, js, err := cred.Dial()
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
||||
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
@@ -154,35 +81,73 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
var buildModules []string
|
||||
for _, l := range delivery.AskLanguages {
|
||||
if m, err := delivery.BuildModuleOf(l); err == nil {
|
||||
buildModules = append(buildModules, m)
|
||||
}
|
||||
}
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
CheckAfter: bound,
|
||||
Toolchains: tools,
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 6)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||
b.Log = func(e delivery.BuildLogEntry) {
|
||||
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||
// is asked since R4g, ADR 0524 §7).
|
||||
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
||||
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||
case kv == nil:
|
||||
default:
|
||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||
b.Answered = says.Answered
|
||||
@@ -196,11 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
watch.put = gate.Toolchain
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
|
||||
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -212,28 +177,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// module.
|
||||
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||
// The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the
|
||||
// builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says
|
||||
// so; it still asks held at its start and at every module value.
|
||||
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
|
||||
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
|
||||
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
|
||||
default:
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -262,23 +225,93 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
if candidates != nil {
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
}
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
// delivery.SourceHistory of them: what a candidate's base revision names.
|
||||
type sourceHeads struct {
|
||||
mu sync.Mutex
|
||||
byRev map[uint64]map[string]string
|
||||
}
|
||||
|
||||
// put takes one revision in and answers whether it is new.
|
||||
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||
if !opOf(e) {
|
||||
return false
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.byRev == nil {
|
||||
s.byRev = map[uint64]map[string]string{}
|
||||
}
|
||||
if _, had := s.byRev[e.Revision()]; had {
|
||||
return false
|
||||
}
|
||||
s.byRev[e.Revision()] = src.Heads
|
||||
for len(s.byRev) > delivery.SourceHistory {
|
||||
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heads, ok := s.byRev[rev]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||
}
|
||||
for r, head := range heads {
|
||||
if delivery.SameRepository(r, repository) {
|
||||
return head, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||
// and opened again at the next event that needs it where there is one.
|
||||
|
||||
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||
// and a candidate's preview waits, saying why.
|
||||
type trunkSource struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
heads *sourceHeads
|
||||
moved func(context.Context)
|
||||
errs chan<- error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
}
|
||||
|
||||
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||
func (t *trunkSource) ensure() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.opened {
|
||||
return nil
|
||||
}
|
||||
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
}
|
||||
t.opened = true
|
||||
go func() {
|
||||
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if t.heads.put(e) && t.moved != nil {
|
||||
t.moved(t.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||
func (t *trunkSource) start() {
|
||||
if err := t.ensure(); err != nil {
|
||||
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||
}
|
||||
}
|
||||
|
||||
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if err := t.ensure(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t.heads.head(ctx, rev, repository)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
|
||||
type queueReader interface {
|
||||
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
|
||||
}
|
||||
|
||||
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
|
||||
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
|
||||
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
|
||||
// instance's) or deleted is left.
|
||||
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := build.DecodeQueueAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
a.Error = err.Error()
|
||||
}
|
||||
a.Queue, a.Answered = &q, time.Now().UTC()
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
|
||||
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
|
||||
}
|
||||
|
||||
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
|
||||
// longer for the builder's answer.
|
||||
const buildAgainBound = 15 * time.Second
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeBuild {
|
||||
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
|
||||
go builtAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeAnnounce {
|
||||
announcedAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
} else {
|
||||
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
|
||||
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
|
||||
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
case res.Failed != "":
|
||||
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
|
||||
default:
|
||||
r.Pending = res.Revision == 0
|
||||
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
|
||||
if res.Pending != "" {
|
||||
r.Answer = res.Pending
|
||||
if res.Revision != 0 {
|
||||
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
|
||||
// unchanged, and the value is written back with the new revision, or why nothing was done.
|
||||
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
|
||||
if res.Refused != "" {
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
} else {
|
||||
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key
|
||||
// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential.
|
||||
// The id makes a repeated append of one entry one entry.
|
||||
func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
|
||||
data, err := delivery.EncodeBuildLogEntry(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := delivery.BuildLogKey(e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key)
|
||||
msg.Data = data
|
||||
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
|
||||
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
|
||||
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
|
||||
// issue 774).
|
||||
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return false
|
||||
}
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
@@ -9,26 +9,31 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
|
||||
// eventsStream is the stream every module's and role's events are on, where the grants part makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-sdk buscompose EventsStream).
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
|
||||
const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held"
|
||||
|
||||
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
|
||||
type toolCaller func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
|
||||
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
|
||||
// the key is then left done, and the controller's take-in still refuses a gone blob.
|
||||
// storedVia is build.Builder's Stored: it asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2, issue 1937)
|
||||
// and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: the key
|
||||
// is then left done, and asked again at the next module value or artifacts-gone.
|
||||
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
|
||||
return func(_ context.Context, references []string) ([]string, error) {
|
||||
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
|
||||
body := map[string]any{"references": references}
|
||||
raw, err := call(SeatHeld, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
|
||||
return nil, fmt.Errorf("asking held: %w", err)
|
||||
}
|
||||
a, err := delivery.DecodeHeldAnswer(raw, references)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,11 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -62,11 +64,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,12 +3,15 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
@@ -20,3 +23,5 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 h1:RtAFbUJRP1qO9AIMM2ue+cXNzqj7WZ7IOlSqNvttwuU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
|
||||
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
|
||||
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
|
||||
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
|
||||
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
|
||||
|
||||
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
|
||||
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
|
||||
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch cur.State {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
|
||||
default:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
|
||||
}
|
||||
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
|
||||
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
|
||||
}
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
|
||||
}
|
||||
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
|
||||
return BuildAgainResult{Revision: nrev}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
key *delivery.BuilderBuild
|
||||
fp string
|
||||
refuse string
|
||||
}{
|
||||
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
|
||||
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
|
||||
{name: "a key with no build is refused", refuse: "has no build key"},
|
||||
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
before := bytes.Clone(r.st.vals[r.key])
|
||||
fp := fp1
|
||||
if c.fp != "" {
|
||||
fp = c.fp
|
||||
}
|
||||
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
|
||||
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
|
||||
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
|
||||
const AskedAgainEvent = "build.asked-again"
|
||||
|
||||
// AskedAgain is the body of AskedAgainEvent.
|
||||
type AskedAgain struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
BuildModule string `json:"build_module"`
|
||||
Language string `json:"language"`
|
||||
Old string `json:"old"`
|
||||
New string `json:"new"`
|
||||
Asking int `json:"asking"`
|
||||
Askings int `json:"askings"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
|
||||
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
|
||||
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
|
||||
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
|
||||
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
|
||||
// five hours, each counted as waiting, so the ask was never asked again).
|
||||
type AskSeen struct {
|
||||
Stream string
|
||||
// Kind is the ask's kind, At when it was asked (its id's time).
|
||||
Kind string
|
||||
At time.Time
|
||||
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
|
||||
Step bool
|
||||
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
|
||||
OnStream bool
|
||||
PutAt time.Time
|
||||
// Ahead are the messages of its subject before it on the stream.
|
||||
Ahead []Queued
|
||||
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
|
||||
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
|
||||
Last *Taken
|
||||
}
|
||||
|
||||
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
|
||||
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
|
||||
type Queued struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
PutAt time.Time `json:"put"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
|
||||
type Taken struct {
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
Node string `json:"node"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
|
||||
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
|
||||
// its worker had none of waiting's kind left to give.
|
||||
func Proves(taken, waiting string) bool {
|
||||
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
|
||||
return t >= 0 && w >= 0 && t >= w
|
||||
}
|
||||
|
||||
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
|
||||
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
|
||||
// its age, and is not counted.
|
||||
func (s AskSeen) Waits(q Queued) bool {
|
||||
return q.Step == "" && !s.passed(q.At)
|
||||
}
|
||||
|
||||
func (s AskSeen) passed(at time.Time) bool {
|
||||
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
|
||||
}
|
||||
|
||||
// Waiting is how many messages ahead still wait to be delivered (Waits).
|
||||
func (s AskSeen) Waiting() int {
|
||||
n := 0
|
||||
for _, q := range s.Ahead {
|
||||
if s.Waits(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
|
||||
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
|
||||
// ADR 0525 §9).
|
||||
func (s AskSeen) Why() string {
|
||||
switch {
|
||||
case s.Step:
|
||||
return ""
|
||||
case !s.OnStream:
|
||||
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
|
||||
case s.Waiting() > 0 || !s.passed(s.at()):
|
||||
return ""
|
||||
default:
|
||||
behind := ""
|
||||
if n := len(s.Ahead); n > 0 {
|
||||
behind = fmt.Sprintf(" behind %d given up on or taken", n)
|
||||
}
|
||||
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
|
||||
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
|
||||
func (s AskSeen) at() time.Time {
|
||||
if s.At.IsZero() {
|
||||
return s.PutAt
|
||||
}
|
||||
return s.At
|
||||
}
|
||||
|
||||
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
|
||||
func AskedAt(id string) time.Time {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(ms)
|
||||
}
|
||||
|
||||
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
|
||||
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
|
||||
type Dropper interface {
|
||||
Drop(ctx context.Context, a delivery.StepAsk) error
|
||||
}
|
||||
|
||||
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
|
||||
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
d, ok := b.Gone.(Dropper)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
prior := ask
|
||||
prior.ID = old
|
||||
if err := d.Drop(ctx, prior); err != nil {
|
||||
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
|
||||
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
|
||||
// ask again, so a lone node off the agreed toolchain cannot spin on it.
|
||||
const (
|
||||
DefaultDeclineBound = 30 * time.Minute
|
||||
DefaultDeclinePause = 5 * time.Second
|
||||
)
|
||||
|
||||
func (b *Builder) declineBound() time.Duration {
|
||||
if b.DeclineBound > 0 {
|
||||
return b.DeclineBound
|
||||
}
|
||||
return DefaultDeclineBound
|
||||
}
|
||||
|
||||
func (b *Builder) declinePause() time.Duration {
|
||||
if b.DeclinePause != 0 {
|
||||
return max(b.DeclinePause, 0)
|
||||
}
|
||||
return DefaultDeclinePause
|
||||
}
|
||||
|
||||
// declinedWords names the nodes that declined an ask and why, in the order of their names.
|
||||
func declinedWords(by map[string]string) string {
|
||||
nodes := make([]string, 0, len(by))
|
||||
for n := range by {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var parts []string
|
||||
for _, n := range nodes {
|
||||
parts = append(parts, n+": "+by[n])
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
}
|
||||
return DefaultAskings
|
||||
}
|
||||
|
||||
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
|
||||
|
||||
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
|
||||
func askingOf(step, buildModule string) int {
|
||||
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
|
||||
if m[1] == buildModule {
|
||||
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
|
||||
// languages whose lane waits, none when no lane of this builder waits on the key.
|
||||
func (b *Builder) AskAgain(key, why string) []string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var asked []string
|
||||
for language, c := range b.waiting[key] {
|
||||
select {
|
||||
case c <- why:
|
||||
default: // one is already asked
|
||||
}
|
||||
asked = append(asked, language)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
|
||||
func (b *Builder) Waiting() map[string][]string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for key, ls := range b.waiting {
|
||||
for l := range ls {
|
||||
out[key] = append(out[key], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Builder) listen(key, language string) chan string {
|
||||
b.init()
|
||||
c := make(chan string, 1)
|
||||
b.mu.Lock()
|
||||
if b.waiting[key] == nil {
|
||||
b.waiting[key] = map[string]chan string{}
|
||||
}
|
||||
b.waiting[key][language] = c
|
||||
b.mu.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
b.mu.Lock()
|
||||
if b.waiting[key][language] == c {
|
||||
delete(b.waiting[key], language)
|
||||
if len(b.waiting[key]) == 0 {
|
||||
delete(b.waiting, key)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
|
||||
errDeclined = errors.New("declined")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
|
||||
func (h byHand) Error() string { return "asked again by hand: " + h.why }
|
||||
|
||||
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
|
||||
// asked it again (AskAgain); it answers the step and the ask that has it.
|
||||
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
|
||||
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
var firstDecline time.Time
|
||||
declines := map[string]string{} // node -> why it declined, the last
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errDeclined):
|
||||
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
|
||||
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
|
||||
// checker's decline loop is, issue 890).
|
||||
now := b.now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
declines[step.Node] = why
|
||||
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
|
||||
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
|
||||
}
|
||||
// The pause is a context's bound, not a timer (ADR 0437 §8).
|
||||
pause, stop := context.WithTimeout(ctx, b.declinePause())
|
||||
<-pause.Done()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return step, ask, ctx.Err()
|
||||
}
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
}
|
||||
asking++
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
|
||||
if b.Said != nil {
|
||||
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
|
||||
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
|
||||
}
|
||||
default:
|
||||
return step, ask, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
|
||||
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
|
||||
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
|
||||
hand, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
go func() {
|
||||
select {
|
||||
case why := <-again:
|
||||
cancel(byHand{why})
|
||||
case <-hand.Done():
|
||||
}
|
||||
}()
|
||||
wait := hand
|
||||
if b.Gone != nil && b.CheckAfter > 0 {
|
||||
var stop context.CancelFunc
|
||||
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
|
||||
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
|
||||
// after a pause, within DeclineBound.
|
||||
return step, step.Reason, errDeclined
|
||||
}
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
var h byHand
|
||||
if errors.As(context.Cause(hand), &h) {
|
||||
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
|
||||
}
|
||||
if errors.Is(context.Cause(wait), errCheck) {
|
||||
why, gerr := b.Gone.Gone(ctx, ask)
|
||||
if gerr != nil {
|
||||
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
|
||||
return step, "", errCheck
|
||||
}
|
||||
if why == "" {
|
||||
return step, "", errCheck
|
||||
}
|
||||
return step, why, errAgain
|
||||
}
|
||||
return step, "", err
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The build runner a build node runs an ask with is mesh-build-go's (novox/hq ADR 0593 §1), another repository's,
|
||||
// which the builder does not import. These tests run an ask as the builder needs it run: fetch once at the job's
|
||||
// fetch arguments with its package versions, build or copy each artifact, answer each one's digest and reference. They
|
||||
// test the builder, not the runner.
|
||||
|
||||
// stepFunc is one step of a build module, as its tool answers it.
|
||||
type stepFunc func(ctx context.Context, args map[string]any) (any, error)
|
||||
|
||||
// askSteps are a build module's steps an ask is run with.
|
||||
type askSteps struct{ Fetch, Build, Copy stepFunc }
|
||||
|
||||
// runAsk runs a build ask with steps and answers its delivery.BuildAnswer, or the error that fails the step.
|
||||
func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build == nil {
|
||||
return nil, fmt.Errorf("the ask %s holds no build job", ask.ID)
|
||||
}
|
||||
job := *ask.Build
|
||||
fetch := job.Fetch
|
||||
if len(ask.Packages) > 0 && fetch != nil {
|
||||
fetch = maps.Clone(fetch)
|
||||
fetch["packages"] = maps.Clone(ask.Packages)
|
||||
}
|
||||
answer := delivery.BuildAnswer{}
|
||||
workspace := ""
|
||||
for _, a := range job.Artifacts {
|
||||
var out any
|
||||
var err error
|
||||
if a.From != "" {
|
||||
if steps.Copy == nil {
|
||||
return nil, fmt.Errorf("this build module copies no artifact (%s)", a.Artifact)
|
||||
}
|
||||
out, err = steps.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
|
||||
} else {
|
||||
if workspace == "" {
|
||||
raw, err := asRaw(steps.Fetch(ctx, fetch))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Merge string `json:"merge"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
||||
return nil, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
||||
}
|
||||
if _, candidate := fetch["base"]; candidate && (len(f.Conflicts) > 0 || f.Merge == "") {
|
||||
return nil, fmt.Errorf("the head does not merge into the trunk: %s", strings.Join(f.Conflicts, ", "))
|
||||
}
|
||||
workspace = f.Workspace
|
||||
}
|
||||
args := map[string]any{"workspace": workspace, "module": job.Path, "artifact": a.Artifact}
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
out, err = steps.Build(ctx, args)
|
||||
}
|
||||
raw, err := asRaw(out, err)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, raw)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func asRaw(v any, err error) (json.RawMessage, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
|
||||
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
|
||||
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
|
||||
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
|
||||
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
|
||||
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
|
||||
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
|
||||
|
||||
// BuildAgainResult is the builder's answer to a request to build a key again.
|
||||
type BuildAgainResult struct {
|
||||
// Refused is why nothing was done, "" when the key was written building as a new revision.
|
||||
Refused string
|
||||
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
|
||||
Revision uint64
|
||||
Asks map[string]string
|
||||
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
|
||||
Pending string
|
||||
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
|
||||
// toolchain not agreed): the new revision was written, and nothing was asked.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// handWatch follows one request to build a key again until its asks are put.
|
||||
type handWatch struct {
|
||||
rev uint64
|
||||
lanes int
|
||||
asks map[string]string
|
||||
done chan struct{}
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
func (w *handWatch) settle() {
|
||||
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
|
||||
w.shut = true
|
||||
close(w.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) hand(key string, f func(w *handWatch)) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if w := b.hands[key]; w != nil {
|
||||
f(w)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) handClaimed(key string, rev uint64) {
|
||||
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handLanes(key string, n int) {
|
||||
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handAsk(key, language, id string) {
|
||||
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
|
||||
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
|
||||
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
|
||||
// ctx is the builder's own context: the build it starts lives on it.
|
||||
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
|
||||
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
|
||||
case cur.State != delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
|
||||
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
|
||||
}
|
||||
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
|
||||
// and the watch and the look are registered before anything else can take the key.
|
||||
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
|
||||
b.mu.Lock()
|
||||
m, known := b.known[module]
|
||||
switch {
|
||||
case !known || m.FP2 != fp:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
|
||||
case m.Manifest == "":
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
|
||||
case b.inflight[key] || b.hands[key] != nil:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
|
||||
}
|
||||
b.hands[key] = w
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
if b.hands[key] == w {
|
||||
delete(b.hands, key)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
|
||||
b.startOnce(ctx, key, module, m)
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
|
||||
defer stop()
|
||||
select {
|
||||
case <-w.done:
|
||||
case <-wait.Done():
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
// The look ended before every ask was put: say how the key stands, not that it was asked.
|
||||
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
|
||||
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
|
||||
switch cur.State {
|
||||
case delivery.BuildFailed:
|
||||
res.Failed = cur.Error
|
||||
case delivery.BuildDone:
|
||||
default:
|
||||
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
|
||||
}
|
||||
}
|
||||
}
|
||||
case !shut:
|
||||
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
|
||||
@@ -68,14 +68,22 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
continue
|
||||
}
|
||||
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest)}
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
|
||||
if len(cm.Paths) > 0 {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
}
|
||||
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
||||
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
||||
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
||||
base, err := b.trunkHead(ctx, c.Base, p.Repository)
|
||||
if err != nil {
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
|
||||
continue
|
||||
}
|
||||
b.mu.Lock()
|
||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, name, m)
|
||||
@@ -106,3 +114,18 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
||||
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if b.Trunk == nil {
|
||||
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
||||
}
|
||||
head, err := b.Trunk(ctx, rev, repository)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
||||
}
|
||||
if head == "" {
|
||||
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,13 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -40,11 +44,74 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
|
||||
}
|
||||
|
||||
const (
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
||||
// commit its fingerprint first appeared at.
|
||||
trunkHead = "3333333333333333333333333333333333333333"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
||||
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
||||
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w struct {
|
||||
Source struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
} `json:"source"`
|
||||
Candidate json.RawMessage `json:"candidate"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
||||
}
|
||||
|
||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
||||
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
||||
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
||||
{
|
||||
rev, heads, c, raw := indexerCandidate(t)
|
||||
p := c.Pulls[0]
|
||||
cm := p.Fps["avahi"]
|
||||
ir := newRig(t, goBundle)
|
||||
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
||||
if at != rev {
|
||||
return "", fmt.Errorf("source revision %d is not heard", at)
|
||||
}
|
||||
return heads[repository], nil
|
||||
}
|
||||
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
|
||||
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ir.b.Module(context.Background(), "avahi", value)
|
||||
ir.b.Wait()
|
||||
ir.bm.mu.Lock()
|
||||
ir.bm.calls = nil
|
||||
ir.bm.mu.Unlock()
|
||||
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
||||
ir.b.Wait()
|
||||
fetches := ir.bm.callsOf("build_go_fetch")
|
||||
if len(fetches) != 1 {
|
||||
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
|
||||
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
||||
}
|
||||
}
|
||||
r := newRig(t, goBundle)
|
||||
trunkModule(t, r)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||
@@ -58,8 +125,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
|
||||
t.Fatalf("%d fetches, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
||||
}
|
||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
||||
t.Errorf("build args = %+v", b)
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
|
||||
@@ -61,8 +61,8 @@ type ClaimOptions struct {
|
||||
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
|
||||
// clause 6). Nil means none is.
|
||||
Published func(ref string) bool
|
||||
// Stored answers which of the references given the artifact store no longer holds (the controller's verb held,
|
||||
// mesh-sdk delivery.HeldVerb; novox/hq issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// Stored answers which of the references given the artifact store no longer holds (the seat mesh-artifact-sweep's verb held,
|
||||
// novox/hq ADR 0571 §2, issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
|
||||
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
|
||||
@@ -29,7 +29,7 @@ func captured(t *testing.T, name string) json.RawMessage {
|
||||
return step.Result
|
||||
}
|
||||
|
||||
// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's
|
||||
// The captured image steps, run as a pulled ask runs them (runAsk), answer what the builder reads: the trunk's
|
||||
// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a).
|
||||
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
|
||||
for _, tc := range []struct {
|
||||
@@ -37,7 +37,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
publish bool
|
||||
ref bool
|
||||
}{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} {
|
||||
steps := buildstep.AskSteps{
|
||||
steps := askSteps{
|
||||
Fetch: func(context.Context, map[string]any) (any, error) {
|
||||
return json.RawMessage(captured(t, "fetch.json")), nil
|
||||
},
|
||||
@@ -47,7 +47,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish,
|
||||
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, ask)
|
||||
out, err := runAsk(context.Background(), steps, ask)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.build, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry,
|
||||
// where the indexer keeps them on the module.
|
||||
func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"}
|
||||
r.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
step := r.answer(a)
|
||||
var answer map[string]any
|
||||
if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" {
|
||||
answer["imports"] = imports
|
||||
step.Result, _ = json.Marshal(answer)
|
||||
}
|
||||
return step
|
||||
}
|
||||
var logged []delivery.BuildLogEntry
|
||||
var mu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() }
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") {
|
||||
t.Fatalf("done key %+v, want the imports", v)
|
||||
}
|
||||
if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 {
|
||||
t.Fatalf("log %+v, want the imports on the end alone", logged)
|
||||
}
|
||||
}
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
|
||||
|
||||
@@ -3,8 +3,8 @@ package build
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
|
||||
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
|
||||
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
|
||||
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
|
||||
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
|
||||
// there, as builder_ask_again does through reasks.
|
||||
const (
|
||||
SeatQueueTool = "builder_seat_queue"
|
||||
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
|
||||
QueuesState = "queues"
|
||||
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
|
||||
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
|
||||
DefaultQueueLimit = 50
|
||||
MaxQueueLimit = 500
|
||||
)
|
||||
|
||||
// QueueNote says what the read cannot show.
|
||||
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
|
||||
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
|
||||
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
|
||||
|
||||
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
|
||||
type QueueAsk struct {
|
||||
V int `json:"v"`
|
||||
Limit int `json:"limit"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Queue *SeatQueue `json:"queue,omitempty"`
|
||||
}
|
||||
|
||||
// SeatQueue is what the builder read of one build seat's stream.
|
||||
type SeatQueue struct {
|
||||
Language string `json:"language"`
|
||||
Stream string `json:"stream"`
|
||||
Messages uint64 `json:"messages"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
Kinds []KindQueue `json:"kinds"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
|
||||
// last a build node took of the kind (or of one pulled after it), and each message.
|
||||
type KindQueue struct {
|
||||
Kind string `json:"kind"`
|
||||
Read int `json:"read"`
|
||||
Waiting int `json:"waiting"`
|
||||
GivenUp int `json:"given_up_or_taken"`
|
||||
Last *Taken `json:"last_taken,omitempty"`
|
||||
Messages []QueuedState `json:"messages"`
|
||||
}
|
||||
|
||||
// QueuedState is one message as the tool shows it.
|
||||
type QueuedState struct {
|
||||
Queued
|
||||
AgeSeconds int64 `json:"age_s"`
|
||||
Waits bool `json:"waits"`
|
||||
}
|
||||
|
||||
// NewKindQueue is seen's messages ahead as one kind's queue at now.
|
||||
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
|
||||
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
|
||||
for _, q := range seen.Ahead {
|
||||
w := seen.Waits(q)
|
||||
if w {
|
||||
k.Waiting++
|
||||
} else {
|
||||
k.GivenUp++
|
||||
}
|
||||
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
|
||||
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
|
||||
if _, err := delivery.AskStream(language); err != nil {
|
||||
return "", nil, fmt.Errorf("language: %w", err)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = DefaultQueueLimit
|
||||
}
|
||||
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
|
||||
return language, data, err
|
||||
}
|
||||
|
||||
// DecodeQueueAsk reads a queues value.
|
||||
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
|
||||
var a QueueAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
|
||||
func LanguageOf(arg string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
|
||||
}
|
||||
@@ -305,10 +305,9 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test
|
||||
r := newRig(t, tsPackage)
|
||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
||||
paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
|
||||
tree, err := delivery.NamedTree(manifestD, paths)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it,
|
||||
// never computes it.
|
||||
const tree = delivery.NamedTreePrefix + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281"
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".",
|
||||
Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}})
|
||||
if err != nil {
|
||||
@@ -379,8 +378,9 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
||||
t.Fatal("no fetch")
|
||||
}
|
||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
|
||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
||||
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
||||
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
@@ -6,15 +6,17 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
||||
@@ -50,9 +52,33 @@ type Builder struct {
|
||||
// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
|
||||
// rests on it. Nil tells nobody.
|
||||
Answered func(buildModule string)
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||
Gone interface {
|
||||
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
|
||||
}
|
||||
CheckAfter time.Duration
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
|
||||
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
|
||||
DeclineBound, DeclinePause time.Duration
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
|
||||
Log func(delivery.BuildLogEntry)
|
||||
// Now is the builder's clock: of the log, and of the decline bound (DeclineBound runs from the first decline); nil is
|
||||
// time.Now. Node is the node the builder runs on, which asked the build.
|
||||
Now func() time.Time
|
||||
Node string
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -72,6 +98,10 @@ type Builder struct {
|
||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -88,7 +118,8 @@ func (b *Builder) init() {
|
||||
b.sem = make(chan struct{}, limit)
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held = map[string]bool{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -231,8 +262,8 @@ func (b *Builder) storedOnce(key string) func(ctx context.Context, references []
|
||||
}
|
||||
}
|
||||
|
||||
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
|
||||
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
|
||||
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
|
||||
// builder: what is built rests on the store's answer, never on the event alone.
|
||||
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
|
||||
@@ -269,6 +300,13 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
||||
for _, k := range sortedKeys(known) {
|
||||
b.consider(ctx, k, known[k])
|
||||
}
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
|
||||
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
||||
// revision it was previewed onto had no trunk head to be merged onto.
|
||||
func (b *Builder) TrunkMoved(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
cands := make([]delivery.Candidate, 0, len(b.cands))
|
||||
for _, c := range b.cands {
|
||||
@@ -348,20 +386,45 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
|
||||
}()
|
||||
}
|
||||
|
||||
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
|
||||
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
|
||||
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
defer b.handEnded(key)
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
if planErr == nil && !nothing {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
@@ -376,6 +439,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now()))
|
||||
if cl.Unasked != nil {
|
||||
b.logf("%v", cl.Unasked)
|
||||
}
|
||||
@@ -425,6 +490,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
b.logf("%s: the build could not be written: %v", cl.Key, err)
|
||||
return
|
||||
}
|
||||
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
}
|
||||
@@ -445,10 +511,11 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
@@ -468,6 +535,7 @@ type lane struct {
|
||||
toolchain *delivery.ToolchainUsed
|
||||
digests map[string]string // artifact -> digest
|
||||
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
|
||||
imports []string // the mesh packages the lane's programs import, as its build module answered them
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -482,7 +550,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
for _, bm := range plan.Modules() {
|
||||
if prior != nil && len(only) > 0 && !only[bm] {
|
||||
kept = append(kept, bm)
|
||||
@@ -492,10 +561,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
sitting = append(sitting, SittingOutWords(language, ag)...)
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
@@ -511,6 +582,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||
}
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.handLanes(key, len(lanes))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
@@ -589,10 +663,14 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
}
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(sitting) > 0 {
|
||||
step += " (" + strings.Join(sitting, "; ") + ")"
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
|
||||
Imports: importsOf(lanes)}
|
||||
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
|
||||
// A reference the build module answered that does not name the artifact and its digest is no reference.
|
||||
return failed("%s%v", delivery.NotStored, err)
|
||||
@@ -663,12 +741,14 @@ type askNotes struct {
|
||||
b *Builder
|
||||
cl *Claim
|
||||
resumed map[string]string
|
||||
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
|
||||
step string
|
||||
}
|
||||
|
||||
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||
n := &askNotes{b: b, cl: cl}
|
||||
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||
n.resumed = maps.Clone(cl.Build.Asks)
|
||||
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -691,7 +771,11 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
v.Asks = map[string]string{}
|
||||
}
|
||||
v.Asks[language] = id
|
||||
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
|
||||
return err
|
||||
}
|
||||
n.b.handAsk(n.cl.Key, language, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||
@@ -738,6 +822,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
job.Fetch = req.Args()
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
|
||||
if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
}
|
||||
if id := asked.resume(language); id != "" {
|
||||
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||
if b.Held != nil {
|
||||
@@ -766,7 +855,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
return
|
||||
}
|
||||
}
|
||||
step, err := b.Asks.Wait(ctx, ask)
|
||||
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
|
||||
if err != nil {
|
||||
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||
return
|
||||
@@ -781,6 +870,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
|
||||
return
|
||||
}
|
||||
l.imports = answer.Imports
|
||||
built := map[string]delivery.AskBuilt{}
|
||||
for _, a := range answer.Artifacts {
|
||||
built[a.Artifact] = a
|
||||
@@ -810,3 +900,87 @@ func needsFetch(tasks []Task) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *Builder) now() time.Time {
|
||||
if b.Now != nil {
|
||||
return b.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// logBuild tells the log an entry, said in the journal when it cannot be one.
|
||||
func (b *Builder) logBuild(e delivery.BuildLogEntry) {
|
||||
if b.Log == nil {
|
||||
return
|
||||
}
|
||||
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
|
||||
b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err)
|
||||
return
|
||||
}
|
||||
b.Log(e)
|
||||
}
|
||||
|
||||
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
|
||||
func (b *Builder) refOf(module, fp string) string {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.via[key].Ref
|
||||
}
|
||||
|
||||
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
|
||||
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
|
||||
// that asked it and the time.
|
||||
func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry {
|
||||
dir := m.Path
|
||||
if dir == "" {
|
||||
dir = delivery.RootPath
|
||||
}
|
||||
return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10),
|
||||
Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()}
|
||||
}
|
||||
|
||||
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
|
||||
// was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build.
|
||||
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
|
||||
e := LogStarted(module, m, ref, by, at)
|
||||
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
|
||||
if result.Manifest != "" {
|
||||
e.Manifest = result.Manifest
|
||||
}
|
||||
if m.Inputs2 != nil {
|
||||
e.Against = append([]string(nil), m.Inputs2.Bases...)
|
||||
}
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
|
||||
return e
|
||||
}
|
||||
|
||||
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
|
||||
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
|
||||
// kept one reports only what it built.
|
||||
func importsOf(lanes []*lane) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lanes {
|
||||
for _, p := range l.imports {
|
||||
seen[p] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
|
||||
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
|
||||
func SittingOutWords(language string, a delivery.Agreement) []string {
|
||||
var out []string
|
||||
for _, s := range a.SittingOut {
|
||||
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -5,13 +5,16 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -136,7 +139,7 @@ type rig struct {
|
||||
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
|
||||
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
|
||||
|
||||
// answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the
|
||||
// answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the
|
||||
// rig's pulling node, and answers the step that node writes.
|
||||
func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
node := r.puller
|
||||
@@ -145,7 +148,7 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
}
|
||||
module, _ := delivery.BuildModuleOf(a.Language)
|
||||
runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}
|
||||
tool := func(name string) buildstep.StepFunc {
|
||||
tool := func(name string) stepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||
raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args)
|
||||
if err != nil {
|
||||
@@ -154,8 +157,8 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||
return raw, nil
|
||||
}
|
||||
}
|
||||
steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
steps := askSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
|
||||
out, err := runAsk(context.Background(), steps, a)
|
||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
@@ -176,6 +179,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||
r.b = &Builder{
|
||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
||||
}
|
||||
var err error
|
||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||
@@ -217,8 +221,52 @@ var (
|
||||
)
|
||||
|
||||
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
|
||||
t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog)
|
||||
r := newRig(t, goBundle, archiveA, tsBundle)
|
||||
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
|
||||
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
|
||||
defer func() {
|
||||
if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
|
||||
t.Errorf("the key does not say ace sits out: %s", st)
|
||||
}
|
||||
}()
|
||||
var logged []delivery.BuildLogEntry
|
||||
var lmu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
|
||||
v := r.consider(t, "modules/m")
|
||||
// 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read
|
||||
// from the two.
|
||||
if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent ||
|
||||
logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" ||
|
||||
logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" ||
|
||||
logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" ||
|
||||
logged[0].FP != fp1 || logged[0].Module != "m" {
|
||||
t.Fatalf("logged = %+v", logged)
|
||||
}
|
||||
for _, e := range logged {
|
||||
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
|
||||
t.Errorf("an entry the log refuses: %v", err)
|
||||
}
|
||||
}
|
||||
if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok {
|
||||
t.Error("no duration from the two entries")
|
||||
}
|
||||
// builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more.
|
||||
var page stdio.LogPage
|
||||
for i, e := range logged {
|
||||
raw, _ := delivery.EncodeBuildLogEntry(e)
|
||||
page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At})
|
||||
}
|
||||
page.Next = 3
|
||||
since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At)
|
||||
if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More {
|
||||
t.Errorf("builds since = %+v, %v, want the one end and more", since, err)
|
||||
}
|
||||
page.Entries[0].Value = []byte(`{"v":0}`)
|
||||
if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil {
|
||||
t.Error("an entry that does not read was skipped, want it refused")
|
||||
}
|
||||
if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
|
||||
t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
|
||||
}
|
||||
@@ -261,9 +309,16 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
||||
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil // no node has pulled it yet
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
|
||||
sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
var asks []delivery.StepAsk
|
||||
for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
|
||||
for deadline := time.Now().Add(10 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask was put")
|
||||
}
|
||||
@@ -273,6 +328,9 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
|
||||
t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
|
||||
}
|
||||
if !reflect.DeepEqual(a.Packages, sdk) {
|
||||
t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
|
||||
t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
|
||||
}
|
||||
@@ -507,6 +565,7 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
|
||||
t.Run("a recipe that moves builds the module again under its new fingerprint", aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint)
|
||||
t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
|
||||
}
|
||||
// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
|
||||
@@ -531,6 +590,45 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0603 §3 (D9): a source revision changing the Go build module's recipe and a Go module builds the module at once with
|
||||
// the recipe the nodes run and again, a second key under the new fingerprint, once the agreement moves to the new recipe;
|
||||
// the first key is neither rebuilt nor waited on.
|
||||
func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.consider(t, "modules/m")
|
||||
calls := len(r.bm.calls)
|
||||
fp2 := "src2:" + hex64('2')
|
||||
for _, n := range []string{"ace", "novox"} {
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.tools.Put(buildrun.GoBuildModule, n+".go", b, false)
|
||||
}
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if len(r.bm.calls) != calls {
|
||||
t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls)
|
||||
}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp2,
|
||||
Manifest: mdigest('a'), Path: "modules/m", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
second, err := delivery.BuilderBuildKey("m", fp2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := valueOf(t, r.st, second); v.State != delivery.BuildDone || v.Toolchains["go"].Recipe != "2" {
|
||||
t.Errorf("second build = %+v, want done with recipe 2", v)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
|
||||
t.Errorf("first build = %+v, want it left done", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
|
||||
r := newRig(t, goBundle)
|
||||
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
|
||||
@@ -615,27 +713,459 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||
// drop from their stream (Dropper).
|
||||
type goneAsks struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
why func(a delivery.StepAsk) string
|
||||
dropped []string
|
||||
}
|
||||
|
||||
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.dropped = append(g.dropped, a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
|
||||
g.mu.Lock()
|
||||
g.n++
|
||||
g.mu.Unlock()
|
||||
return g.why(a), nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) asked() int {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.n
|
||||
}
|
||||
|
||||
// waitForAsks waits until n asks were put.
|
||||
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(10 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask was put")
|
||||
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
return r.asks.Asks()
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
|
||||
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Run("waits in its stream: never asked again", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
waitForAsks(t, r, 1)
|
||||
// Five check bounds passed (five reads of whether it can be answered), so the ask waited past the step bound: an event
|
||||
// to wait on, where a pause of 100 ms said nothing on a loaded machine.
|
||||
for deadline := time.Now().Add(10 * time.Second); gone.asked() < 5; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("whether the ask can still be answered was read %d times within 10 s, want 5", gone.asked())
|
||||
}
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
if gone.asked() == 0 {
|
||||
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
// The builder's clock steps a second at each read, so the bound of a minute ends the declines after many asks however
|
||||
// slow the machine (a bound of 150 ms of wall clock let a loaded one put fewer than DefaultAskings).
|
||||
var clock atomic.Int64
|
||||
start := time.Now()
|
||||
r.b.Now = func() time.Time { return start.Add(time.Duration(clock.Add(1)) * time.Second) }
|
||||
r.b.DeclinePause, r.b.DeclineBound = -1, time.Minute
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
|
||||
waitForAsks(t, r, 1)
|
||||
for ended, deadline := 0, time.Now().Add(20*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
|
||||
r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
|
||||
ended++
|
||||
}
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
|
||||
t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
deaf bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
|
||||
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
|
||||
// is refused, "" when it is built again as a new revision and its ask id answered.
|
||||
again bool
|
||||
key *delivery.BuilderBuild
|
||||
known string
|
||||
refuse string
|
||||
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
|
||||
// the error the key then holds when its build ends before its asks are put.
|
||||
edit func(m *delivery.Module)
|
||||
busy int
|
||||
failed string
|
||||
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
|
||||
during bool
|
||||
}{
|
||||
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
|
||||
edit: func(m *delivery.Module) {
|
||||
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
|
||||
},
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
|
||||
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
|
||||
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
|
||||
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
|
||||
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
|
||||
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
|
||||
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
if c.again {
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.known != "" {
|
||||
m.FP2 = c.known
|
||||
}
|
||||
if c.edit != nil {
|
||||
c.edit(&m)
|
||||
}
|
||||
r.b.init()
|
||||
var stored atomic.Int64
|
||||
if c.during {
|
||||
r.asks.Answer = nil
|
||||
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
|
||||
}
|
||||
switch c.busy {
|
||||
case 1:
|
||||
r.b.inflight[r.key] = true
|
||||
case 2:
|
||||
r.b.hands[r.key] = &handWatch{}
|
||||
}
|
||||
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
|
||||
log := &logged{}
|
||||
r.b.Logf = log.logf
|
||||
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
|
||||
if c.during {
|
||||
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
|
||||
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
|
||||
}
|
||||
return
|
||||
}
|
||||
r.b.Wait()
|
||||
after := r.st.revs[r.key]
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
|
||||
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.failed != "" {
|
||||
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
|
||||
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
asks := r.asks.Asks()
|
||||
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
|
||||
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
|
||||
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
|
||||
}
|
||||
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
|
||||
t.Errorf("the why is not logged: %v", log.lines)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
var mu sync.Mutex
|
||||
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||
first := ""
|
||||
if c.step != "" {
|
||||
first = "mvl0frv18d1960"
|
||||
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
dead := &buildrun.DeadLetters{}
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
var heard *buildrun.HeardEnds
|
||||
if c.deaf {
|
||||
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead && !c.deaf {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.dead {
|
||||
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||
}
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(10 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no lane of the key waits")
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.deaf {
|
||||
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
|
||||
for deadline := time.Now().Add(10 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask is waited on")
|
||||
}
|
||||
}
|
||||
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
|
||||
}
|
||||
if c.done && !c.deaf {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
}
|
||||
last := asks[len(asks)-1]
|
||||
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
|
||||
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
|
||||
}
|
||||
r.asks.End(r.answer(last))
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); n != c.puts {
|
||||
t.Errorf("%d asks put, want %d", n, c.puts)
|
||||
}
|
||||
if c.done && v.State != delivery.BuildDone {
|
||||
t.Errorf("the key = %+v, want done from the ask asked again", v)
|
||||
}
|
||||
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
|
||||
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
taken := func(kind string, at time.Time) *Taken {
|
||||
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
|
||||
}
|
||||
later := taken(delivery.AskDelivery, put.Add(time.Minute))
|
||||
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
|
||||
ahead := func(n int, step string) []Queued {
|
||||
var qs []Queued
|
||||
for i := range n {
|
||||
at := put.Add(-time.Duration(n-i) * time.Minute)
|
||||
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
|
||||
}
|
||||
return qs
|
||||
}
|
||||
seen := func(qs []Queued, last *Taken) AskSeen {
|
||||
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
seen AskSeen
|
||||
want string // a part of the why, "" when it can be answered
|
||||
waiting int
|
||||
}{
|
||||
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
|
||||
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
|
||||
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
|
||||
{"first in line, nothing taken since", seen(nil, nil), "", 0},
|
||||
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
|
||||
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
|
||||
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
|
||||
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
|
||||
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
|
||||
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
|
||||
} {
|
||||
got := c.seen.Why()
|
||||
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
|
||||
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
|
||||
}
|
||||
}
|
||||
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
|
||||
t.Errorf("AskedAt does not read an ask id's time")
|
||||
}
|
||||
})
|
||||
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
|
||||
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
state := func(vals map[string]string) StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) {
|
||||
var ks []string
|
||||
for k := range vals {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
return ks, nil
|
||||
}, Get: func(k string) (json.RawMessage, time.Time, error) {
|
||||
if v, ok := vals[k]; ok {
|
||||
return json.RawMessage(v), at, nil
|
||||
}
|
||||
return nil, time.Time{}, nil
|
||||
}}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||
done, _ := delivery.BuilderBuildKey("d", fp1)
|
||||
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
|
||||
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
}
|
||||
a := stuck[0].Asks
|
||||
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
|
||||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
|
||||
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
|
||||
}
|
||||
|
||||
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
|
||||
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
|
||||
// only for the rows kept.
|
||||
var gets, written, inFlight, maxFlight atomic.Int64
|
||||
vals := map[string]string{}
|
||||
for i := 0; i < 3000; i++ {
|
||||
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
|
||||
vals[k] = string(finished)
|
||||
if (i/100)%10 == 0 {
|
||||
vals[k] = string(building)
|
||||
}
|
||||
}
|
||||
slow := state(vals)
|
||||
get := slow.Get
|
||||
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
|
||||
gets.Add(1)
|
||||
n := inFlight.Add(1)
|
||||
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
raw, _, err := get(k)
|
||||
return raw, time.Time{}, err
|
||||
}
|
||||
slow.Written = func(string) time.Time { written.Add(1); return at }
|
||||
for _, row := range []struct {
|
||||
module string
|
||||
stuck int
|
||||
gets int64
|
||||
cut bool
|
||||
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
|
||||
gets.Store(0)
|
||||
written.Store(0)
|
||||
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cut := out["cut"]
|
||||
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
|
||||
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
|
||||
}
|
||||
}
|
||||
if maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// heldAsks says an ask is held when its id is among held.
|
||||
@@ -697,3 +1227,10 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
|
||||
}
|
||||
}
|
||||
|
||||
// silentWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type silentWatch struct{}
|
||||
|
||||
func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more.
|
||||
const BuildsSinceLimit = 500
|
||||
|
||||
// LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since.
|
||||
type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error)
|
||||
|
||||
// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on,
|
||||
// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again
|
||||
// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that
|
||||
// skipped it would register too little.
|
||||
func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) {
|
||||
out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}}
|
||||
page, err := read(since, "", BuildsSinceLimit)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err)
|
||||
}
|
||||
if page == nil {
|
||||
return out, nil
|
||||
}
|
||||
var entries []delivery.BuildLogEntry
|
||||
for _, le := range page.Entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(le.Value)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
if ended := delivery.EndedSince(entries, since); ended != nil {
|
||||
out.Builds = ended
|
||||
}
|
||||
out.More = page.Next != 0
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
||||
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
||||
// builder's state reasks, which the builder watches and answers there.
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
|
||||
BuildAgainTool = "builder_build_again"
|
||||
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
|
||||
// of Mode ModeAnnounce.
|
||||
AnnounceAgainTool = "builder_announce_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
)
|
||||
|
||||
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
||||
// read it, its answer.
|
||||
type ReAsk struct {
|
||||
V int `json:"v"`
|
||||
Why string `json:"why"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
|
||||
// revision (building) and the ask ids it put by language.
|
||||
Refused bool `json:"refused,omitempty"`
|
||||
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
|
||||
// Failed is the error of a key written building that failed before its asks were put.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
Revision uint64 `json:"revision,omitempty"`
|
||||
Asks map[string]string `json:"asks,omitempty"`
|
||||
}
|
||||
|
||||
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
|
||||
const ModeBuild = "build"
|
||||
|
||||
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
|
||||
const ModeAnnounce = "announce"
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
if r.V != 1 {
|
||||
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
||||
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the build is asked again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
|
||||
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the key is built again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
|
||||
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the done build is announced again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, time.Time, error)
|
||||
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
||||
// only for a row they keep, and only when Get did not say.
|
||||
Written func(key string) time.Time
|
||||
}
|
||||
|
||||
// StuckAsk is one ask a key waits on.
|
||||
type StuckAsk struct {
|
||||
Language string `json:"language"`
|
||||
BuildModule string `json:"build_module"`
|
||||
ID string `json:"id"`
|
||||
Asking int `json:"asking"`
|
||||
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
||||
// state on its node since its start.
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
// Stuck is one build key that waits on an ask.
|
||||
type Stuck struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
State string `json:"state"`
|
||||
Step string `json:"step"`
|
||||
By string `json:"by"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
Age string `json:"age,omitempty"`
|
||||
Asks []StuckAsk `json:"asks"`
|
||||
ReAsk *ReAsk `json:"reask,omitempty"`
|
||||
Unasked bool `json:"unasked,omitempty"`
|
||||
NoneStep int `json:"asks_with_no_step"`
|
||||
}
|
||||
|
||||
// StuckReader reads what builder_stuck answers.
|
||||
type StuckReader struct {
|
||||
Builds StateRead
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||
Hearing StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
||||
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
||||
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
||||
const (
|
||||
ReadWorkers = 16
|
||||
MaxReads = 1000
|
||||
)
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||
func (r StuckReader) deadLetters() map[string]any {
|
||||
if r.Hearing.Get == nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||
}
|
||||
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||
}
|
||||
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
defer func() {
|
||||
if r.Logf != nil {
|
||||
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}()
|
||||
keys, err := r.Builds.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now
|
||||
if r.Now != nil {
|
||||
now = r.Now
|
||||
}
|
||||
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
||||
stepKeys := map[string]map[string]string{}
|
||||
for bm, s := range r.Steps {
|
||||
ks, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
||||
}
|
||||
byID := make(map[string]string, len(ks))
|
||||
for _, k := range ks {
|
||||
if _, id, err := delivery.ParseStepKey(k); err == nil {
|
||||
byID[id] = k
|
||||
}
|
||||
}
|
||||
stepKeys[bm] = byID
|
||||
}
|
||||
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
||||
// parallel, at most MaxReads of them.
|
||||
type parsed struct{ key, mod, fp string }
|
||||
var matching []parsed
|
||||
for _, key := range keys {
|
||||
mod, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||
if err != nil || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
matching = append(matching, parsed{key, mod, fp})
|
||||
}
|
||||
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
||||
cut := len(matching) > MaxReads
|
||||
total := len(matching)
|
||||
if cut {
|
||||
matching = matching[:MaxReads]
|
||||
}
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
ats := make([]time.Time, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, at, err := r.Builds.Get(matching[i].key)
|
||||
if err == nil {
|
||||
values[i], ats[i] = raw, at
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reads += len(matching)
|
||||
out := []Stuck{}
|
||||
for i, m := range matching {
|
||||
key, mod, fp := m.key, m.mod, m.fp
|
||||
raw, at := values[i], ats[i]
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key)
|
||||
}
|
||||
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
||||
if !at.IsZero() {
|
||||
s.Age = now().Sub(at).Round(time.Second).String()
|
||||
}
|
||||
for language, id := range v.Asks {
|
||||
bm, _ := delivery.BuildModuleOf(language)
|
||||
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
|
||||
if t, ok := idTime(id); ok {
|
||||
a.Asked = t
|
||||
}
|
||||
if st, ok := r.Steps[bm]; ok {
|
||||
if k, ok := stepKeys[bm][id]; ok {
|
||||
reads++
|
||||
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
||||
if step, err := delivery.DecodeStep(raw); err == nil {
|
||||
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.Step == "none" {
|
||||
s.NoneStep++
|
||||
}
|
||||
s.Asks = append(s.Asks, a)
|
||||
}
|
||||
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
||||
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
|
||||
if r.ReAsks.Get != nil {
|
||||
reads++
|
||||
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
||||
if ra, err := DecodeReAsk(raw); err == nil {
|
||||
s.ReAsk = &ra
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if !out[i].Since.Equal(out[j].Since) {
|
||||
return out[i].Since.Before(out[j].Since)
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
answer["dead_letters"] = r.deadLetters()
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// StuckBuildModules are the build modules whose steps builder_stuck reads.
|
||||
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
|
||||
|
||||
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
||||
func idTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
@@ -23,24 +25,31 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Logs []struct {
|
||||
Name string `json:"name"`
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
} `json:"logs"`
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Pattern string `json:"pattern"`
|
||||
@@ -84,13 +93,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
"mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
@@ -98,12 +107,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
}
|
||||
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
|
||||
// other tool; it holds no capacity.
|
||||
if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 {
|
||||
// 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955).
|
||||
if want := []string{"seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
|
||||
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
|
||||
}
|
||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
|
||||
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
|
||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
// starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the
|
||||
// controller's no longer (1126/B9, ADR 0590).
|
||||
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -112,7 +123,18 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
// 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream.
|
||||
if len(m.Logs) != 1 || m.Logs[0].Name != delivery.BuilderLog || m.Logs[0].MaxAgeDays != delivery.BuilderLogDays {
|
||||
t.Errorf("logs = %+v, want %s kept %d days", m.Logs, delivery.BuilderLog, delivery.BuilderLogDays)
|
||||
}
|
||||
for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
@@ -127,7 +149,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
||||
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
@@ -149,8 +171,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||
|
||||
@@ -2,17 +2,34 @@
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_announce_again",
|
||||
"builder_ask_again",
|
||||
"builder_build_again",
|
||||
"builder_builds_since",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-artifact-sweep.artifacts-gone",
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
"mesh-controller.artifacts-gone"
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.held"
|
||||
"seat:mesh-artifact-sweep.held"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "instances", "history": 1 }
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
],
|
||||
"logs": [
|
||||
{ "name": "build-log", "max-age-days": 30 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
@@ -20,7 +37,8 @@
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules"
|
||||
@@ -30,7 +48,6 @@
|
||||
"node-build-image",
|
||||
"node-build-node"
|
||||
],
|
||||
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
@@ -77,6 +94,17 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder",
|
||||
"binary": "mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder-tools",
|
||||
"binary": "mesh-delivery-builder-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,22 +1,51 @@
|
||||
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
|
||||
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
|
||||
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
// state and reads. It writes only the tries (the try's tools) and, asking a check again by hand, one verdict. The checker is the other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
|
||||
var reads atomic.Int64
|
||||
|
||||
// cutAnswer says in the answer that it was cut, and how to narrow it.
|
||||
func cutAnswer(answer map[string]any, cut string) map[string]any {
|
||||
if cut != "" {
|
||||
answer["cut"], answer["note"] = true, cut
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
|
||||
func logged(tools []stdio.Tool) []stdio.Tool {
|
||||
for i := range tools {
|
||||
name, run := tools[i].Name, tools[i].Run
|
||||
tools[i].Run = func(args map[string]any) (any, error) {
|
||||
reads.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(args)
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) check.StateRead {
|
||||
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
reads.Add(1)
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
@@ -25,10 +54,36 @@ func read(s stdio.KeptState) check.StateRead {
|
||||
}}
|
||||
}
|
||||
|
||||
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||
type runsOf struct{ modules check.StateRead }
|
||||
|
||||
func (r runsOf) CommitOf(module string) string {
|
||||
if r.modules.Get == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := r.modules.Get(module)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// stateStore is a state the tools write: the tries.
|
||||
type stateStore struct {
|
||||
check.StateRead
|
||||
create func(key string, value []byte) error
|
||||
entry func(key string) (json.RawMessage, uint64, error)
|
||||
update func(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||
return s.update(key, value, revision)
|
||||
}
|
||||
|
||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||
@@ -38,19 +93,21 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||
type Trying struct {
|
||||
Tries check.TryStore
|
||||
Ender check.TryEnder
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
||||
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||
func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -78,7 +135,8 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -87,24 +145,61 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if tr.Ender == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||
}
|
||||
now := time.Now
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try,
|
||||
// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them.
|
||||
func SeatTryTools(tr Trying) []stdio.Tool {
|
||||
verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"}
|
||||
var out []stdio.Tool
|
||||
for _, t := range TryTools(tr) {
|
||||
t.Name = Seat + "." + verbs[t.Name]
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Tools are the read tools over the checker's states.
|
||||
func Tools(r check.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: check.VerdictsReadTool,
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, every failure it names one line each (every member's failed tests, refused steps and conflicts, the manifests' refusals and the composition), the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
group, _ := args["group"].(string)
|
||||
vs, err := r.Verdicts(group)
|
||||
return map[string]any{"verdicts": vs}, err
|
||||
vs, cut, err := r.Verdicts(group)
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
Description: "The verdict that judges one pull request, every failure it names one line each (all of them, not only the first the status line shows), and the full report of its status mesh/check. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
||||
@@ -122,8 +217,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
level, _ := args["level"].(string)
|
||||
digest, _ := args["digest"].(string)
|
||||
kept, err := r.Kept(level, digest)
|
||||
return map[string]any{"kept": kept}, err
|
||||
kept, cut, err := r.Kept(level, digest)
|
||||
return cutAnswer(map[string]any{"kept": kept}, cut), err
|
||||
}},
|
||||
{Name: check.PreviewsReadTool,
|
||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||
@@ -138,20 +233,43 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
if head != "" && len(head) < 7 {
|
||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||
}
|
||||
ps, bad, err := r.Previews(repository, head)
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
ps, bad, cut, err := r.Previews(repository, head)
|
||||
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. builds_not_ended names, of each running check, the builder's builds of " +
|
||||
"its candidate's changed modules that are neither done nor failed: the check waits on them before it composes, within its build-wait-bound " +
|
||||
"(novox/hq ADR 0603 §10); a failed one fails the check: rebuild it with the builder's builder_build_again, and the check asks itself again when the build is done. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
}
|
||||
|
||||
// RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose
|
||||
// wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w.
|
||||
func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{{Name: check.AskAgainTool,
|
||||
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
|
||||
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
|
||||
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
|
||||
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
|
||||
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return r.AskAgain(w, str(args, "group"), str(args, "why"))
|
||||
}}}
|
||||
}
|
||||
|
||||
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
|
||||
// duration.
|
||||
func stepBound() time.Duration {
|
||||
@@ -165,7 +283,10 @@ func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
@@ -177,14 +298,35 @@ func main() {
|
||||
return s.Heads, err
|
||||
}
|
||||
r.Trunks = trunks
|
||||
trying := Trying{
|
||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
||||
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := tries.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
verdicts := stdio.State("verdicts")
|
||||
written := stateStore{entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := verdicts.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,14 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("tools = %v, want %v", names, want)
|
||||
}
|
||||
// The recovery tool beside them asks a lost check again; with nothing to write it refuses.
|
||||
recovery := RecoveryTools(check.Reader{}, nil)
|
||||
if len(recovery) != 1 || recovery[0].Name != check.AskAgainTool || recovery[0].Description == "" {
|
||||
t.Errorf("recovery tools = %+v", recovery)
|
||||
}
|
||||
if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil {
|
||||
t.Errorf("%s answered with no state to write", check.AskAgainTool)
|
||||
}
|
||||
// With nothing to read from a tool says so and changes nothing.
|
||||
for _, tl := range Tools(check.Reader{}) {
|
||||
if tl.Name == check.StatusReadTool {
|
||||
@@ -53,13 +61,19 @@ func (m *memStore) Create(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||
m.vals[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
||||
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
@@ -74,6 +88,32 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike.
|
||||
verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" {
|
||||
t.Fatalf("verbs = %+v", verbs)
|
||||
}
|
||||
again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
if err != nil || again.(map[string]any)["id"] != id {
|
||||
t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id)
|
||||
}
|
||||
if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) {
|
||||
t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw)
|
||||
}
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
t.Errorf("end = %v, %v", ended, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||
t.Error("an ended try was ended again")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||
t.Errorf("hq: %v", err)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -124,9 +125,61 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
return parseManifest(kept.JSON)
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
|
||||
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
|
||||
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
|
||||
// parseManifest decodes a manifest the indexer kept, as every reader does (novox/hq ADR 0578 §3): judged by nothing here,
|
||||
// a field it does not know set aside. The checker judges a change's manifests with the indexer's judge, run as a step
|
||||
// (judge.go).
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
return catalogue.Decode([]byte(raw))
|
||||
}
|
||||
|
||||
// awaitedKV is a bucket that may appear after the start (process.AwaitBucket) as a check.KV: a call while it is not on the
|
||||
// bus fails saying so.
|
||||
type awaitedKV struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (a awaitedKV) kv() (kvStore, error) {
|
||||
kv := a.b.KV()
|
||||
if kv == nil {
|
||||
return kvStore{}, errors.New("the bucket is not on the bus yet")
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
}
|
||||
|
||||
func (a awaitedKV) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return s.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Keys(ctx context.Context) ([]string, error) {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Keys(ctx)
|
||||
}
|
||||
|
||||
func (a awaitedKV) Delete(ctx context.Context, key string) error {
|
||||
s, err := a.kv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Delete(ctx, key)
|
||||
}
|
||||
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// assignerWrote raises the assigner's compositions bucket, the assigner online in the watchman's presence, and its verdict
|
||||
// under the check's key, as the assigner leaves them on the bus.
|
||||
func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerdict) {
|
||||
t.Helper()
|
||||
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, pkey, on); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeCompositionVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(b.ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
|
||||
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
|
||||
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
state, owed, words string
|
||||
}{
|
||||
{delivery.CompositionPass, "success", "composition passed"},
|
||||
{delivery.CompositionFail, "failure", "g14 runs web, which the change removes from its source"},
|
||||
} {
|
||||
b := startBed(t)
|
||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
|
||||
v := delivery.CompositionVerdict{Group: group, Heads: heads, State: c.state, SDK: "v0.1.14-x", Machines: []string{"g14"}}
|
||||
if c.state == delivery.CompositionFail {
|
||||
v.Machine, v.Words = "g14", c.words
|
||||
}
|
||||
b.assignerWrote(t, key, v)
|
||||
b.start(t, nil)
|
||||
got := b.verdict(t, key, ended)
|
||||
if got.Composition == nil || got.Composition.State != c.state {
|
||||
t.Fatalf("%s: the verdict's composition is %+v", c.state, got.Composition)
|
||||
}
|
||||
o := b.owed(t, 7, head1, c.owed)
|
||||
if !strings.Contains(o.Text, c.words) || strings.Contains(o.Text, "not yet run") {
|
||||
t.Errorf("%s: the line says %q", c.state, o.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user