Compare commits
877
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7be95905f2 | ||
|
|
04fe2e30fe | ||
|
|
52ad5af1e9 | ||
|
|
cd16358b21 | ||
|
|
0c2ead025d | ||
|
|
5ab9bd9acc | ||
|
|
dc33ae26bb | ||
|
|
a096397928 | ||
|
|
5548fff81f | ||
|
|
52fa062e92 | ||
|
|
ef32d60a5e | ||
|
|
cfd2afe39c | ||
|
|
21daeb8893 | ||
|
|
ae1bf2098d | ||
|
|
f09bd2668a | ||
|
|
82156f8128 | ||
|
|
ffa58e51c4 | ||
|
|
cc1034630f | ||
|
|
73cb9ba690 | ||
|
|
fcac1fc5d0 | ||
|
|
db899d8ad4 | ||
|
|
4ca6e478a6 | ||
|
|
564fd5e0bb | ||
|
|
dc6dfef6f0 | ||
|
|
8eee17a368 | ||
|
|
2b79ee1593 | ||
|
|
9931d4f900 | ||
|
|
18136ce063 | ||
|
|
a5495bb15e | ||
|
|
5d3164b5b7 | ||
|
|
96d8136e2b | ||
|
|
c45f1621fb | ||
|
|
fd2cb13961 | ||
|
|
915a54f136 | ||
|
|
1c0309df23 | ||
|
|
7dd2891f32 | ||
|
|
b7e364e47b | ||
|
|
dca2545eb7 | ||
|
|
bd5852b817 | ||
|
|
1bfd54425b | ||
|
|
c6c86801ca | ||
|
|
33feb148ce | ||
|
|
f44442097c | ||
|
|
1f765ef17b | ||
|
|
203454b1a9 | ||
|
|
b691428ffc | ||
|
|
a41a7c2862 | ||
|
|
314a996ed9 | ||
|
|
7e103f3221 | ||
|
|
c37a7317b7 | ||
|
|
b2c9e60ac3 | ||
|
|
b77b70a66c | ||
|
|
6f417c88e9 | ||
|
|
7f5f175b05 | ||
|
|
1f33a4c04f | ||
|
|
49c656a4ce | ||
|
|
9005a2ee8e | ||
|
|
b914974b1d | ||
|
|
3f72078853 | ||
|
|
31a484e457 | ||
|
|
29afbe0c5a | ||
|
|
abaddd3ef8 | ||
|
|
d79b9f1518 | ||
|
|
7e25aa75f8 | ||
|
|
df5a8895e2 | ||
|
|
81f866c9e5 | ||
|
|
d151033dae | ||
|
|
3c742145d7 | ||
|
|
d557c0c2af | ||
|
|
0f65240139 | ||
|
|
3f25e659ba | ||
|
|
a85c9d4032 | ||
|
|
c11c90348e | ||
|
|
1b177d5912 | ||
|
|
0c3b9833cd | ||
|
|
d1d092980a | ||
|
|
241cfd581a | ||
|
|
ce7669d433 | ||
|
|
473451f7e6 | ||
|
|
5f090cd99f | ||
|
|
a5e10c70e2 | ||
|
|
b0a614779d | ||
|
|
c9d7b1365a | ||
|
|
342da2c625 | ||
|
|
e611103ec9 | ||
|
|
dde4108504 | ||
|
|
81ea73a45f | ||
|
|
d68e71700a | ||
|
|
c0489000ba | ||
|
|
eac4dca2a7 | ||
|
|
7cf8ea380e | ||
|
|
61d6424c44 | ||
|
|
a75d2d07bf | ||
|
|
3ff3987aa2 | ||
|
|
a63f4e001b | ||
|
|
abfe37e690 | ||
|
|
56ca9a9791 | ||
|
|
380412d94f | ||
|
|
1146a6a37f | ||
|
|
eb02389d45 | ||
|
|
6545469a2b | ||
|
|
8c50d77b04 | ||
|
|
bb1ab9f10d | ||
|
|
f756718324 | ||
|
|
5e6ebefd47 | ||
|
|
726592b74c | ||
|
|
b6a7e0bad3 | ||
|
|
b65b617361 | ||
|
|
474ef0335a | ||
|
|
ed3c4a2297 | ||
|
|
ce6c39e46c | ||
|
|
d6d15a5497 | ||
|
|
80ff1afacc | ||
|
|
5e94328066 | ||
|
|
3e90604eb4 | ||
|
|
205a9256b8 | ||
|
|
14cd44b98e | ||
|
|
67caa79640 | ||
|
|
9ab1c3b8f3 | ||
|
|
6b0ccc5d27 | ||
|
|
7d58b2e56f | ||
|
|
f838a6c34c | ||
|
|
381934396b | ||
|
|
6ebc63dce3 | ||
|
|
a343fddfe7 | ||
|
|
c8ba486318 | ||
|
|
92ba593639 | ||
|
|
b45f12f70f | ||
|
|
c8cc249a90 | ||
|
|
b3724ca0f4 | ||
|
|
ba54f82a70 | ||
|
|
76b1ea11a9 | ||
|
|
8e926283e3 | ||
|
|
13d960a120 | ||
|
|
29162d170c | ||
|
|
aac7476cfe | ||
|
|
515688cf6f | ||
|
|
83caf486b6 | ||
|
|
bcff47d257 | ||
|
|
b76b9c58f4 | ||
|
|
40c2870a71 | ||
|
|
9a7cc2b632 | ||
|
|
38ad9ed84d | ||
|
|
d096ba37e3 | ||
|
|
7ed668fa53 | ||
|
|
9b2b2a934c | ||
|
|
84989dafb2 | ||
|
|
d13efa8abe | ||
|
|
40ad2650bd | ||
|
|
43a88ef96c | ||
|
|
b9bc44024d | ||
|
|
477d351fc0 | ||
|
|
8f89dd29f5 | ||
|
|
346b37d623 | ||
|
|
a43b722dbb | ||
|
|
8d43ce0cb9 | ||
|
|
c623987fcc | ||
|
|
1aba89e8ba | ||
|
|
7c9da7d0cb | ||
|
|
427885013e | ||
|
|
02ec2f00d9 | ||
|
|
6ea33475b6 | ||
|
|
cc5f803f87 | ||
|
|
fad9f059ee | ||
|
|
d719a69b7d | ||
|
|
905a055a1e | ||
|
|
48aa36120d | ||
|
|
e216e94029 | ||
|
|
dc72860ce0 | ||
|
|
62e8274bf9 | ||
|
|
8bd66112f0 | ||
|
|
8617b521a6 | ||
|
|
8813c4e9e9 | ||
|
|
b00cf73d77 | ||
|
|
7ffec1b56b | ||
|
|
592637e6d7 | ||
|
|
3fce8700bf | ||
|
|
3a647a8e2d | ||
|
|
7dbd440a4e | ||
|
|
9acf583daf | ||
|
|
f420a8213a | ||
|
|
e7d9849efa | ||
|
|
f6e7f4210b | ||
|
|
1058b0eec6 | ||
|
|
125d6382ea | ||
|
|
d2d3ebbf44 | ||
|
|
278310cc67 | ||
|
|
1767d7b4e9 | ||
|
|
ac99946fcb | ||
|
|
4749c686eb | ||
|
|
1f5b0f813d | ||
|
|
fe5fa6ac94 | ||
|
|
99b25a7db4 | ||
|
|
a559b0e7c9 | ||
|
|
7606d515fd | ||
|
|
b40745534f | ||
|
|
c1ec75789f | ||
|
|
75208ffcba | ||
|
|
6046ebe2e0 | ||
|
|
ff7db33720 | ||
|
|
2df32b7892 | ||
|
|
7e597a623e | ||
|
|
6a847a8d9c | ||
|
|
dc83ef3e27 | ||
|
|
600c13b9e9 | ||
|
|
5396dc9b11 | ||
|
|
425dbaef99 | ||
|
|
714c2c1ccc | ||
|
|
e583dce741 | ||
|
|
5e76fdf725 | ||
|
|
438c8740a6 | ||
|
|
05d9f90bdb | ||
|
|
f22945c26a | ||
|
|
e137df7189 | ||
|
|
6c66bf9a5a | ||
|
|
b79fd19c84 | ||
|
|
70bb8f7f51 | ||
|
|
e196064e51 | ||
|
|
2eaa41a420 | ||
|
|
90ff867e7a | ||
|
|
77fb6c4ab1 | ||
|
|
bda61ae64c | ||
|
|
5547fd028e | ||
|
|
d51d389c1b | ||
|
|
c6edd5987f | ||
|
|
9a49b0f4f7 | ||
|
|
47f81051be | ||
|
|
769d701e2b | ||
|
|
ece9d6b18f | ||
|
|
93495203b5 | ||
|
|
5628ca2919 | ||
|
|
070f60404e | ||
|
|
7f855f0e52 | ||
|
|
05c59ae000 | ||
|
|
3dfc18023a | ||
|
|
1489493431 | ||
|
|
e1630b57c6 | ||
|
|
5c4c4bc821 | ||
|
|
e927bb8971 | ||
|
|
599bf62d23 | ||
|
|
0de195c5aa | ||
|
|
3160bb1fb8 | ||
|
|
93efda5db3 | ||
|
|
f01731fd4e | ||
|
|
251745f859 | ||
|
|
d1ec9d9072 | ||
|
|
14254dd0ba | ||
|
|
1e50642e3c | ||
|
|
b0082ec04f | ||
|
|
6b6e4c81f0 | ||
|
|
374d060cc0 | ||
|
|
1f878eec58 | ||
|
|
c9515da86f | ||
|
|
5f7137e676 | ||
|
|
902947a044 | ||
|
|
b95b3ef989 | ||
|
|
0f167db73d | ||
|
|
e5f54c5cdb | ||
|
|
bdb4edd820 | ||
|
|
31c2c5c7fe | ||
|
|
ad0052c6f5 | ||
|
|
aef8632d95 | ||
|
|
dedcc3ead7 | ||
|
|
a40bbec44a | ||
|
|
fe6c5961ae | ||
|
|
588a6ff8d5 | ||
|
|
ced27928d9 | ||
|
|
2cb6971194 | ||
|
|
6a730c33e6 | ||
|
|
d1aab1d08d | ||
|
|
c6b4b6054b | ||
|
|
4df45a3793 | ||
|
|
e2e13ddd29 | ||
|
|
dcf5b7fa8f | ||
|
|
1fe6ade359 | ||
|
|
9e09767a4b | ||
|
|
59443ab3f7 | ||
|
|
92d0ab8602 | ||
|
|
3a31331ad3 | ||
|
|
d1581666d2 | ||
|
|
d4d7895392 | ||
|
|
d6c209a10c | ||
|
|
51ac838732 | ||
|
|
06a52a10e0 | ||
|
|
773e2e33a1 | ||
|
|
d817d85fca | ||
|
|
6d5000871d | ||
|
|
5c7648744d | ||
|
|
8802f16019 | ||
|
|
6087306dfd | ||
|
|
667aa87611 | ||
|
|
0f2fd1a171 | ||
|
|
e1ce5cc8ed | ||
|
|
7794ee9d2d | ||
|
|
396893dcac | ||
|
|
626a6fb358 | ||
|
|
20ee1cae7b | ||
|
|
671dc5a963 | ||
|
|
3f62410c22 | ||
|
|
4af1ba99cf | ||
|
|
91b5e8d086 | ||
|
|
189d75ad40 | ||
|
|
595510ef38 | ||
|
|
04f54d7773 | ||
|
|
07dc14bf53 | ||
|
|
9b3a728f9e | ||
|
|
83e4470757 | ||
|
|
1b1fd9a711 | ||
|
|
47ea130332 | ||
|
|
52f7a5b6f6 | ||
|
|
aee7e4c26c | ||
|
|
64d5490ea2 | ||
|
|
7199824e9d | ||
|
|
c418f49038 | ||
|
|
e29aa8aca9 | ||
|
|
64cba95a10 | ||
|
|
cb26645614 | ||
|
|
b0d2d22002 | ||
|
|
014f7b7c2f | ||
|
|
0fcc614e25 | ||
|
|
e3fed77515 | ||
|
|
94202fb651 | ||
|
|
0418a2b94b | ||
|
|
a926a40eed | ||
|
|
b9cc97470b | ||
|
|
ae91f91e88 | ||
|
|
4664eb1b00 | ||
|
|
d9a3458927 | ||
|
|
b058389d46 | ||
|
|
6c65e82926 | ||
|
|
d40f314e52 | ||
|
|
fc85f7b936 | ||
|
|
37a8f7a83e | ||
|
|
9ef830d224 | ||
|
|
120a0554d6 | ||
|
|
d8a900dac0 | ||
|
|
a62f7b40c1 | ||
|
|
429f03d4cf | ||
|
|
20d59ff4f0 | ||
|
|
12b18721b9 | ||
|
|
3183e0e056 | ||
|
|
30ccad4edc | ||
|
|
4e585f02b1 | ||
|
|
f2d3172eac | ||
|
|
e4781edcd6 | ||
|
|
1db38fcc72 | ||
|
|
eba4309b19 | ||
|
|
a7fc37561e | ||
|
|
e93575ffe7 | ||
|
|
859f88b2f0 | ||
|
|
517a4b3cf1 | ||
|
|
7f88916863 | ||
|
|
5dee4cb263 | ||
|
|
77b683c1c1 | ||
|
|
60e32772a5 | ||
|
|
fd2ef6c7e8 | ||
|
|
71371df160 | ||
|
|
b2d5644c34 | ||
|
|
9598b92c39 | ||
|
|
f00d34673f | ||
|
|
87357d84bd | ||
|
|
08dde53e33 | ||
|
|
c772eb0683 | ||
|
|
a7b2fb0ffa | ||
|
|
ef086e0b75 | ||
|
|
a8adda1a8c | ||
|
|
e1f0c1a1cc | ||
|
|
bf24bb9740 | ||
|
|
814244fd5a | ||
|
|
442b03f61d | ||
|
|
fb23967de3 | ||
|
|
9083287d57 | ||
|
|
07dd8ea842 | ||
|
|
d0436bb38c | ||
|
|
fe51379ee2 | ||
|
|
037cb9555e | ||
|
|
5c5136d6d1 | ||
|
|
c43e7a5d1c | ||
|
|
e04321acae | ||
|
|
4c2914194b | ||
|
|
277c53067e | ||
|
|
227f6e68ce | ||
|
|
521cbf089e | ||
|
|
39a2c6a944 | ||
|
|
577ec05205 | ||
|
|
744204c5ab | ||
|
|
0f41b405ba | ||
|
|
20f0e9baaf | ||
|
|
75c636a5ec | ||
|
|
0cbabe9860 | ||
|
|
d2c9dc7de6 | ||
|
|
53199f16ce | ||
|
|
256eb34a0d | ||
|
|
a47c8e7922 | ||
|
|
16f9ad2950 | ||
|
|
b1f25f91d5 | ||
|
|
d281faeeb7 | ||
|
|
e312999843 | ||
|
|
5dba7fe68b | ||
|
|
f0f28c56a4 | ||
|
|
20a7bd0f3c | ||
|
|
0b1ce08985 | ||
|
|
f053e3b041 | ||
|
|
d428650ce4 | ||
|
|
351eda0867 | ||
|
|
5e3d92ae33 | ||
|
|
e5e5c50e67 | ||
|
|
8167d825ed | ||
|
|
ac95e5628c | ||
|
|
bed9b61d5a | ||
|
|
67ef449230 | ||
|
|
8a52b6f801 | ||
|
|
380d4781e6 | ||
|
|
7534bf5c07 | ||
|
|
08fe40f233 | ||
|
|
27e96a2a1b | ||
|
|
f98d5ec748 | ||
|
|
abb7cfcf5d | ||
|
|
3956c0cf94 | ||
|
|
403fb3068b | ||
|
|
e54bf7f346 | ||
|
|
f95a164941 | ||
|
|
0cf419619e | ||
|
|
8b6f6b22d4 | ||
|
|
9cddb5921a | ||
|
|
842da1be85 | ||
|
|
3090a80a09 | ||
|
|
03c0b627ad | ||
|
|
af5d5f87dd | ||
|
|
fbd75dfac4 | ||
|
|
40bd5cf69d | ||
|
|
a94b275a9c | ||
|
|
12b48debe9 | ||
|
|
05b91a6281 | ||
|
|
ecc87a6510 | ||
|
|
76b2e8dc7d | ||
|
|
6b50f3eb9d | ||
|
|
c9f0beedd1 | ||
|
|
5f4eb03fcf | ||
|
|
20bbf346dc | ||
|
|
0bb24f91a8 | ||
|
|
70ccb470a7 | ||
|
|
e4d51a47f7 | ||
|
|
1ede9a5f11 | ||
|
|
79322692c6 | ||
|
|
037cb74640 | ||
|
|
46f1a54189 | ||
|
|
bdbfa12aa1 | ||
|
|
4923d22ac6 | ||
|
|
4613300bcd | ||
|
|
1cf2ea46ff | ||
|
|
2a93cbcd9b | ||
|
|
0548dc8f80 | ||
|
|
213668df49 | ||
|
|
bb930de377 | ||
|
|
bbdd1741fa | ||
|
|
63bb53bb6b | ||
|
|
cf703c2a7b | ||
|
|
1bd410522e | ||
|
|
79e4688744 | ||
|
|
c9fd571e8e | ||
|
|
8da18c37c5 | ||
|
|
c3d17f6b5a | ||
|
|
e5a79db912 | ||
|
|
f9d403ffea | ||
|
|
a755925024 | ||
|
|
10c806fe6f | ||
|
|
c67bfcdc0f | ||
|
|
ff2211b3a1 | ||
|
|
e94a5d1256 | ||
|
|
b53555c5bf | ||
|
|
3ad2ac455a | ||
|
|
def28437ce | ||
|
|
5009109fad | ||
|
|
dad31f49c5 | ||
|
|
1a3664ad77 | ||
|
|
c2e4d7914a | ||
|
|
fd68792614 | ||
|
|
4611f263f3 | ||
|
|
1ece9449fd | ||
|
|
8d2d490637 | ||
|
|
e102f7dbc6 | ||
|
|
204cf3e16a | ||
|
|
3447dd5f4d | ||
|
|
1b8f377cad | ||
|
|
9ee012bce5 | ||
|
|
d40e922be4 | ||
|
|
b4f0c3ec88 | ||
|
|
abdd70d918 | ||
|
|
d00ee48363 | ||
|
|
1e826ff008 | ||
|
|
ea28f39319 | ||
|
|
930d09cadc | ||
|
|
cbf1dbb15d | ||
|
|
f9e2dc38d8 | ||
|
|
94d3f02f76 | ||
|
|
11a22fdff7 | ||
|
|
90f53608bc | ||
|
|
2bb08b9306 | ||
|
|
ffb5c48dd2 | ||
|
|
6b43adafca | ||
|
|
f8eea0000a | ||
|
|
9a55e6e8c0 | ||
|
|
78e169a8c6 | ||
|
|
f6163349af | ||
|
|
5bf6fd9b01 | ||
|
|
89e0926feb | ||
|
|
c94e8564e6 | ||
|
|
969caea37f | ||
|
|
be741a7a8f | ||
|
|
f79c0a7b78 | ||
|
|
9fe6330d79 | ||
|
|
7ac48fd882 | ||
|
|
3560362330 | ||
|
|
87e9084e15 | ||
|
|
b75dcc947c | ||
|
|
630c099972 | ||
|
|
f9fa0ec2b2 | ||
|
|
83d454217f | ||
|
|
4599d2e60f | ||
|
|
b7b4087095 | ||
|
|
d24d6a0890 | ||
|
|
5042d9c426 | ||
|
|
6e027de597 | ||
|
|
f8bbde9627 | ||
|
|
f006ab50bc | ||
|
|
8e5e5dbb42 | ||
|
|
ce6edcd9b9 | ||
|
|
bf642c45b7 | ||
|
|
64ae24aef7 | ||
|
|
0ea4b60dbd | ||
|
|
5da42d1775 | ||
|
|
0639ec4bca | ||
|
|
ac833a48f8 | ||
|
|
d9596548f9 | ||
|
|
9a7cd5861d | ||
|
|
95b4b308ad | ||
|
|
b413949ea8 | ||
|
|
69d17f139f | ||
|
|
1175fef241 | ||
|
|
d822259cfa | ||
|
|
ea4fe471e2 | ||
|
|
ff7a27f36f | ||
|
|
7de5814a26 | ||
|
|
dfdfd84efe | ||
|
|
68048619d9 | ||
|
|
8f17d59f5c | ||
|
|
50734000c3 | ||
|
|
2bc9ea9377 | ||
|
|
7617dda5f6 | ||
|
|
13db05ce60 | ||
|
|
334bc54767 | ||
|
|
6f8ba0bf69 | ||
|
|
92196dde3a | ||
|
|
16feb56fd4 | ||
|
|
78adc23d70 | ||
|
|
79bca88c8a | ||
|
|
5bca54bb59 | ||
|
|
1751bff172 | ||
|
|
2e03fcb875 | ||
|
|
b3e52ecdef | ||
|
|
1eec825926 | ||
|
|
04500fa29d | ||
|
|
900316e4bd | ||
|
|
233824eeeb | ||
|
|
6d6caf20e1 | ||
|
|
00b452e57b | ||
|
|
e952ab721b | ||
|
|
33143527f2 | ||
|
|
1dd16bc783 | ||
|
|
7ba0eddd92 | ||
|
|
37dd3dac13 | ||
|
|
533a9e4ca9 | ||
|
|
efd8c7b34f | ||
|
|
80cfaa175f | ||
|
|
16b795af70 | ||
|
|
59ca36e55b | ||
|
|
09be0bc27f | ||
|
|
3a47cccbee | ||
|
|
ea106ea8b9 | ||
|
|
ae91c7e73a | ||
|
|
378853a97d | ||
|
|
33e5c3a105 | ||
|
|
91f3ef809c | ||
|
|
9e793e0142 | ||
|
|
65c4dcbf73 | ||
|
|
43ec4017be | ||
|
|
ab8a8ddfc4 | ||
|
|
22f252251c | ||
|
|
bec4d43da6 | ||
|
|
d006132c2a | ||
|
|
763d6a107a | ||
|
|
a79b811113 | ||
|
|
40b6ced375 | ||
|
|
9768d0b67d | ||
|
|
0adcfbbbdb | ||
|
|
109a3e1e49 | ||
|
|
a66583236a | ||
|
|
80a7061efa | ||
|
|
10d87dcc77 | ||
|
|
1cd984a4c7 | ||
|
|
b31924a2f5 | ||
|
|
7654838182 | ||
|
|
3f51d83147 | ||
|
|
0be2b3d01a | ||
|
|
871f75acd3 | ||
|
|
4b75791d3d | ||
|
|
4205f63786 | ||
|
|
2438f4a108 | ||
|
|
9081966682 | ||
|
|
16cb874ccc | ||
|
|
20d6a68bfb | ||
|
|
21bef6c044 | ||
|
|
ae105b4631 | ||
|
|
b58a8b2e7f | ||
|
|
533a076eb3 | ||
|
|
4e33533bd0 | ||
|
|
1f344bd47c | ||
|
|
5b8fa23a8d | ||
|
|
ed78003b23 | ||
|
|
20e966b9cd | ||
|
|
7697ef5988 | ||
|
|
768213b816 | ||
|
|
f9dd20e86d | ||
|
|
a96f873e21 | ||
|
|
ab9d9604b4 | ||
|
|
2a90323af3 | ||
|
|
00e1588ef9 | ||
|
|
a439b8da47 | ||
|
|
a2f4e8b499 | ||
|
|
8e95c2486f | ||
|
|
1878607208 | ||
|
|
930a407a3a | ||
|
|
eadfcd4cf8 | ||
|
|
19fbc076d6 | ||
|
|
1a924f2ca7 | ||
|
|
ebe49b9bdd | ||
|
|
7923ab63b1 | ||
|
|
09224d2cdf | ||
|
|
167a9f2818 | ||
|
|
ad40520e89 | ||
|
|
2b3a9e4a6a | ||
|
|
72ebb4deb6 | ||
|
|
d957a60286 | ||
|
|
ba11ff58dd | ||
|
|
97f68b04ea | ||
|
|
61a11f8c15 | ||
|
|
cd08f1e1c2 | ||
|
|
afa95fd0df | ||
|
|
ef5e5d13e0 | ||
|
|
5388bbce43 | ||
|
|
185fe0ae1d | ||
|
|
8fa1b8aee7 | ||
|
|
c17ccd3070 | ||
|
|
9849c49920 | ||
|
|
4cdf14680e | ||
|
|
bf5f794553 | ||
|
|
34da4fe001 | ||
|
|
2406bb4000 | ||
|
|
db62653472 | ||
|
|
6bb6452603 | ||
|
|
8d93c3ff9f | ||
|
|
f93e8b9750 | ||
|
|
9a8609f770 | ||
|
|
39162a4435 | ||
|
|
1ccde6fc3e | ||
|
|
45f7bf8e01 | ||
|
|
eed0833a29 | ||
|
|
223b096295 | ||
|
|
1987a3915e | ||
|
|
250a0f1b12 | ||
|
|
61843f823d | ||
|
|
44bdbbcbed | ||
|
|
482a618d1e | ||
|
|
1a187445a5 | ||
|
|
8ab75bb727 | ||
|
|
8c4489fb3d | ||
|
|
a2bcb23f0e | ||
|
|
43050bf20a | ||
|
|
8710148ce0 | ||
|
|
6618925841 | ||
|
|
1eb090d407 | ||
|
|
f373cf4ae6 | ||
|
|
e040cc5fc2 | ||
|
|
38f1c2f10a | ||
|
|
bece2421f1 | ||
|
|
fb257f549b | ||
|
|
b9c50ef0c5 | ||
|
|
f7e0619b24 | ||
|
|
d351ec5932 | ||
|
|
4eef3e6c4d | ||
|
|
92c87aac6c | ||
|
|
34310504b5 | ||
|
|
5188d36a5c | ||
|
|
bfea3d36ad | ||
|
|
592278441c | ||
|
|
48ba977270 | ||
|
|
8671d1062b | ||
|
|
7c3b6211b0 | ||
|
|
cd6a452c08 | ||
|
|
7b71766ab4 | ||
|
|
a90c4bfbf5 | ||
|
|
cdcab2370d | ||
|
|
30f67f0471 | ||
|
|
28b5724871 | ||
|
|
298f4de168 | ||
|
|
bf2962dc29 | ||
|
|
c1fc670af4 | ||
|
|
91b65d0180 | ||
|
|
ac0befa9b9 | ||
|
|
4fabf9d26c | ||
|
|
3d5ea26605 | ||
|
|
10b879bc31 | ||
|
|
00de5a7f43 | ||
|
|
0ee747cf1b | ||
|
|
d69ee47321 | ||
|
|
05c765e36d | ||
|
|
7c0b835ce3 | ||
|
|
c44a42fc16 | ||
|
|
5632924c4f | ||
|
|
bbcad0845a | ||
|
|
d0efbee65b | ||
|
|
c5799d2f9b | ||
|
|
19758f44cf | ||
|
|
f5880d449b | ||
|
|
d0e7dee951 | ||
|
|
a5a5d9a16f | ||
|
|
41821c841a | ||
|
|
017451762d | ||
|
|
e7c05c5370 | ||
|
|
070a5dd35b | ||
|
|
57025ccb3b | ||
|
|
fb588e85c0 | ||
|
|
bfb80605c3 | ||
|
|
e4302579ce | ||
|
|
2979bc3b84 | ||
|
|
8f20ca8fdc | ||
|
|
e67f77f8e6 | ||
|
|
7c338230de | ||
|
|
7548314b95 | ||
|
|
e78b3eea0a | ||
|
|
7989ddc227 | ||
|
|
4abb853400 | ||
|
|
bda4abb13e | ||
|
|
63dff99f11 | ||
|
|
e1476cfdf9 | ||
|
|
7c33a2cf4c | ||
|
|
2682a623de | ||
|
|
7d97516c3b | ||
|
|
485c2f10b1 | ||
|
|
bb164bb2b8 | ||
|
|
479b5b5521 | ||
|
|
7d6eeed8ae | ||
|
|
16517579b2 | ||
|
|
4289d5583b | ||
|
|
773f89e9de | ||
|
|
fe3bdb5d10 | ||
|
|
6b5099c8d7 | ||
|
|
901c800c62 | ||
|
|
6a02d80072 | ||
|
|
ba2a7d73b7 | ||
|
|
f1231fccb0 | ||
|
|
310d4817ca | ||
|
|
5126d83722 | ||
|
|
40771c41ab | ||
|
|
7507a250c7 | ||
|
|
260dab75d5 | ||
|
|
f0e7631d11 | ||
|
|
98875eba70 | ||
|
|
c62f8e07fd | ||
|
|
d1ed5fd555 | ||
|
|
ba0d2eeacc | ||
|
|
e0f12c8c37 | ||
|
|
d57fa43bcc | ||
|
|
e0f0a56630 | ||
|
|
0fe5f0aaaf | ||
|
|
9bb02a2cb2 | ||
|
|
d4ed5ae13f | ||
|
|
21523a449f | ||
|
|
ad3d73d9bd | ||
|
|
9be15fe393 | ||
|
|
1f1126d4ec | ||
|
|
af61faf706 | ||
|
|
06ff2dc2c6 | ||
|
|
b0242f68fe | ||
|
|
240015c814 | ||
|
|
61fff7da67 | ||
|
|
c677534e11 | ||
|
|
8e3b44c6fc | ||
|
|
ce839ba602 | ||
|
|
2beb39c371 | ||
|
|
e84e2adff0 | ||
|
|
2394dec1c5 | ||
|
|
e75112baec | ||
|
|
f89f34ce7b | ||
|
|
1506507f21 | ||
|
|
8a340b5297 | ||
|
|
e0632b2c0e | ||
|
|
69c24b5270 | ||
|
|
9496c57d41 | ||
|
|
84e640b7d2 | ||
|
|
c636642864 | ||
|
|
9eb143ab6c | ||
|
|
598125a1d5 | ||
|
|
218d5cb67f | ||
|
|
c625733a33 | ||
|
|
da82c2cf80 | ||
|
|
0071181fb1 | ||
|
|
1b3cc20127 | ||
|
|
5b0925ce3a | ||
|
|
c9f70c9287 | ||
|
|
7ce4720854 | ||
|
|
9a3a6bd3ba | ||
|
|
3cdd4943f0 | ||
|
|
4ee5b45cb8 | ||
|
|
ddee3bf9af | ||
|
|
4a83418c02 | ||
|
|
5f7f6b3e40 | ||
|
|
5b4c454ca2 | ||
|
|
b0756f8b45 | ||
|
|
1695075836 | ||
|
|
306534fdf0 | ||
|
|
0933144535 | ||
|
|
86fb18afb7 | ||
|
|
a6ab2e54cd | ||
|
|
5255ffa0c4 | ||
|
|
b0d263c336 | ||
|
|
ba50db813a | ||
|
|
3303e9170c | ||
|
|
6a91fa15f6 | ||
|
|
7e4c9b9992 | ||
|
|
34e1938df1 | ||
|
|
990f6c6598 | ||
|
|
e5bb4fa9bc | ||
|
|
acb12fea95 | ||
|
|
c37426f64b | ||
|
|
6edb9ab526 | ||
|
|
dc4f1dff39 | ||
|
|
6c47ffcf8f | ||
|
|
0259d77d11 | ||
|
|
1ed86db4aa | ||
|
|
21b53f5dfb | ||
|
|
fa5df62e97 | ||
|
|
54a19f3f64 | ||
|
|
6edfe1da8d | ||
|
|
89e36241cb | ||
|
|
14901e8723 | ||
|
|
66ae3cd918 | ||
|
|
c3fd353abe | ||
|
|
968e1b7493 | ||
|
|
0ad09c7382 | ||
|
|
8604d121eb | ||
|
|
d6d6bd1891 | ||
|
|
e2eb99fc19 | ||
|
|
fc51bf4832 | ||
|
|
3c929d518c | ||
|
|
e383004afc | ||
|
|
27072735e9 | ||
|
|
30f8e7407b | ||
|
|
6458b72062 | ||
|
|
49fbb17db9 | ||
|
|
ff8b27c89b | ||
|
|
7f3aeedfc0 | ||
|
|
eaa903a8ec | ||
|
|
72aba8c415 | ||
|
|
a118867b43 | ||
|
|
69b6fa7393 | ||
|
|
b2bddecde2 | ||
|
|
bac26c8a28 | ||
|
|
536c573223 | ||
|
|
ec58535ca0 | ||
|
|
359c37d17f | ||
|
|
a015b5ed24 | ||
|
|
4c112de4d5 | ||
|
|
21eaa55b27 | ||
|
|
541eaa20fe | ||
|
|
93e222a8a8 |
@@ -1,225 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The waits for an acting part's new version in the steps the assigner holds (design 47 G §266, §268, §276): an entry of the assignment
|
||||
// graph whose acting edge names composing or placing waits, its want held, until the part's version runs on the edge's node. The wait is
|
||||
// kept in the state `waits` under `acting.<node>.<module>.<part>`, said as `wait-held` when it starts and `wait-ended` (ran, released,
|
||||
// replaced) when it ends; `assigner_waits` reads it and `assigner_release` with how `now` lets the entry move with the part's running
|
||||
// version. A node away starts no wait: its clock starts when the node is present (§268).
|
||||
|
||||
// actingOf is the Module.Acting of module: the held acting edges of its entry on a node, those whose wait was not released left out.
|
||||
func (p *passer) actingOf(module string, released map[string]bool) func(node string) []assign.ActingEdge {
|
||||
return func(node string) []assign.ActingEdge {
|
||||
var out []assign.ActingEdge
|
||||
if p.engineOn(node) == module {
|
||||
return nil // the node-engine's own want is held by no acting edge, so it is written first (§268, A-323, A-330)
|
||||
}
|
||||
for _, e := range p.entryEdges(node, module) {
|
||||
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
|
||||
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// engineOn is the module that holds the seat node-engine on node in the revision the pass computed, read from the acting edges'
|
||||
// seat (A-330); "" when none is read.
|
||||
func (p *passer) engineOn(node string) string {
|
||||
if p.graph == nil {
|
||||
return ""
|
||||
}
|
||||
return assigngraph.Engines(slices.Collect(maps.Values(p.graph.computed.Entries)))[node]
|
||||
}
|
||||
|
||||
// entryEdges are the acting edges of the entry of module on node in the revision the pass computed.
|
||||
func (p *passer) entryEdges(node, module string) []seat.Edge {
|
||||
g := p.graph
|
||||
if g == nil {
|
||||
return nil
|
||||
}
|
||||
for _, ref := range g.computed.Revision.Entries {
|
||||
if ref.Node == node && ref.Module == module {
|
||||
var out []seat.Edge
|
||||
for _, e := range g.computed.Entries[ref.Digest].Edges {
|
||||
if e.Kind == "acting" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// actingReleased is the waits released by hand and not yet run, by `<key>@<part's fingerprint>`.
|
||||
func (p *passer) actingReleased() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, raw := range p.w.read(buckets.Waits()) {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Released && (acting.Is(key) || acting.IsOrder(key)) {
|
||||
out[key+"@"+w.Fp] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// actingWaits keeps the acting waits after the pass's wants: it starts the wait of each entry held for a part's version, ends the
|
||||
// wait of one that ran, was replaced or was released, and says each.
|
||||
func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]meshwants.Want) {
|
||||
g := p.graph
|
||||
if g == nil || p.kvs[buckets.Waits()] == nil {
|
||||
return
|
||||
}
|
||||
now := p.clock()
|
||||
kept := p.w.read(buckets.Waits())
|
||||
standing := map[string]bool{}
|
||||
engines := assigngraph.Engines(slices.Collect(maps.Values(g.computed.Entries)))
|
||||
held := map[string]bool{} // `<node>/<module>` of each entry held for a part's version
|
||||
for _, ref := range g.computed.Revision.Entries {
|
||||
entry := g.computed.Entries[ref.Digest]
|
||||
if w, ok := wanted[ref.Module][ref.Node]; ok && w.Fp == entry.Fp {
|
||||
continue // the entry's version is wanted already: nothing moves, so nothing waits
|
||||
}
|
||||
if engines[ref.Node] == ref.Module {
|
||||
continue // the node-engine's own want waits for no acting part (§268, A-323, A-330)
|
||||
}
|
||||
for _, e := range entry.Edges {
|
||||
if e.Kind != "acting" || !acting.Held(e.Step) {
|
||||
continue
|
||||
}
|
||||
key := acting.Key(ref.Node, ref.Module, e.Module)
|
||||
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
|
||||
continue
|
||||
}
|
||||
standing[key] = true
|
||||
held[ref.Node+"/"+ref.Module] = true
|
||||
if raw := kept[key]; raw != nil {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Entry == entry.Fp && w.Fp == e.Fp {
|
||||
p.actingReleaseSaid(ctx, key, w)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !n.Online[ref.Node] {
|
||||
continue // a node away holds only itself, and its clock has not started (§268)
|
||||
}
|
||||
w := acting.Wait{Step: e.Step, Node: ref.Node, Module: ref.Module, Part: e.Module, PartNode: e.Node, Fp: e.Fp, Entry: entry.Fp,
|
||||
Since: now, ClockFrom: now, Bound: acting.BoundOf(e).String()}
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.Fp+"."+entry.Fp); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", ref.Module, ref.Node, e.Module, err)
|
||||
continue
|
||||
}
|
||||
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
}
|
||||
}
|
||||
p.actingCycles(g.computed.Entries, engines, held, n)
|
||||
for key, raw := range kept {
|
||||
if !acting.Is(key) || standing[key] {
|
||||
continue
|
||||
}
|
||||
w, err := acting.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
how := acting.HowRan
|
||||
if w.Released {
|
||||
how = acting.HowReleased
|
||||
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
|
||||
how = acting.HowReplaced
|
||||
}
|
||||
if !w.EndSaid {
|
||||
body := acting.Body(w, assignment.Module())
|
||||
body.How = how
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
|
||||
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.remove(ctx, buckets.Waits(), key)
|
||||
}
|
||||
}
|
||||
|
||||
// actingReleaseSaid says wait-ended of a wait released by hand, once, and keeps its record so the entry is not held again.
|
||||
func (p *passer) actingReleaseSaid(ctx context.Context, key string, w acting.Wait) {
|
||||
if !w.Released || w.EndSaid {
|
||||
return
|
||||
}
|
||||
body := acting.Body(w, assignment.Module())
|
||||
body.How = acting.HowReleased
|
||||
body.By = w.By
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+acting.HowReleased); err != nil {
|
||||
p.logf("the release of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
return
|
||||
}
|
||||
w.EndSaid = true
|
||||
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
}
|
||||
|
||||
// edgeOf is the edge a kept wait stands on, as an acting edge.
|
||||
func (p *passer) edgeOf(w acting.Wait) (assign.ActingEdge, bool) {
|
||||
return assign.ActingEdge{Node: w.PartNode, Part: w.Part, Fp: w.Fp, Step: w.Step}, true
|
||||
}
|
||||
|
||||
// actingCycles writes the finding acting-cycle of each held want whose holding edges lie on a cycle no rule breaks, and clears it of
|
||||
// every other want, a finding a pass before or another process wrote included (design 47 G A-323, A-330). An edge holds while its
|
||||
// step is the assigner's, its part's version does not run and its wait was not released; the node-engine's own want is held by none.
|
||||
func (p *passer) actingCycles(entries map[string]seat.AssignmentEntry, engines map[string]string, held map[string]bool, n assign.Nodes) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
released := p.actingReleased()
|
||||
cycles := assigngraph.HeldCycles(slices.Collect(maps.Values(entries)), func(e seat.AssignmentEntry, edge seat.Edge) bool {
|
||||
return held[e.Node+"/"+e.Module] && engines[e.Node] != e.Module && edge.Kind == "acting" && acting.Held(edge.Step) &&
|
||||
!released[acting.Key(e.Node, e.Module, edge.Module)+"@"+edge.Fp] && !n.Runs(assign.ActingEdge{Node: edge.Node, Part: edge.Module, Fp: edge.Fp, Step: edge.Step})
|
||||
})
|
||||
raised := map[string]bool{}
|
||||
for _, at := range slices.Sorted(maps.Keys(cycles)) {
|
||||
node, module, _ := strings.Cut(at, "/")
|
||||
o := assign.ActingCycleFinding(node, module, cycles[at])
|
||||
key := mcseatKey(o)
|
||||
raised[key] = true
|
||||
if p.said[key+"="+o.Summary] {
|
||||
continue // written already as it stands
|
||||
}
|
||||
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.said[key+"="+o.Summary] = true
|
||||
}
|
||||
if p.findings.Keys == nil || p.findings.Delete == nil {
|
||||
return
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings %s are not read, so none is cleared this pass: %v", assign.KindActingCycle, err)
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
if strings.HasSuffix(key, "."+assign.KindActingCycle) && !raised[key] {
|
||||
if err := p.findings.Delete(key); err != nil {
|
||||
p.logf("the finding %s is not cleared: %v", key, err)
|
||||
continue
|
||||
}
|
||||
for said := range p.said {
|
||||
if strings.HasPrefix(said, key+"=") {
|
||||
delete(p.said, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §266, §268, §276 (AS9): an entry with an acting edge in composing waits for the part's version to run on the edge's node;
|
||||
// the wait is kept, said as wait-held with its bound and clock-from, read by assigner_waits, released by assigner_release with how
|
||||
// `now` and ended when the version runs.
|
||||
func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T) {
|
||||
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, now)
|
||||
var events []string
|
||||
var bodies []meshwants.Wait
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
events = append(events, event)
|
||||
bodies = append(bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
entry := meshwants.AssignmentEntry{Node: "ace", Module: "nats", Fp: fpTest, Edges: []meshwants.Edge{
|
||||
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Seat: "node-engine", Step: "compose", Bound: "15m"},
|
||||
{Kind: "acting", Node: "ace", Module: "indexer", Fp: "src2:idx", Seat: "mesh-catalog", Step: "take-in"}}}
|
||||
p.graph = &graphKeeper{}
|
||||
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
|
||||
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
|
||||
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
|
||||
ctx := context.Background()
|
||||
|
||||
if got := p.actingOf("nats", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
|
||||
t.Fatalf("the assigner holds composing only: %+v", got)
|
||||
}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
p.actingWaits(ctx, n, wanted) // said once
|
||||
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
b := bodies[0]
|
||||
if b.Step != "compose" || b.Part != "mesh-host" || b.Fp != "src2:new" || b.Bound != "15m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(now) || b.Owner == "" {
|
||||
t.Errorf("wait-held %+v", b)
|
||||
}
|
||||
key := acting.Key("ace", "nats", "mesh-host")
|
||||
if _, err := acting.Decode(p.w.read(buckets.Waits())[key]); err != nil {
|
||||
t.Fatalf("the wait is kept: %v", err)
|
||||
}
|
||||
// a release by hand lets the entry move, and ends the wait once
|
||||
w, _ := acting.Decode(p.w.read(buckets.Waits())[key])
|
||||
w.Released = true
|
||||
raw, _ := acting.Encode(w)
|
||||
p.w.put(buckets.Waits(), key, raw)
|
||||
if got := p.actingOf("nats", p.actingReleased())("ace"); len(got) != 0 {
|
||||
t.Errorf("a released wait still holds: %+v", got)
|
||||
}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != acting.HowReleased {
|
||||
t.Fatalf("events %v bodies %+v", events, bodies)
|
||||
}
|
||||
// the version runs: the record goes, said nothing more
|
||||
n.Running["ace"]["mesh-host"] = nodeengine.Running{Fingerprint: "src2:new"}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
if len(p.w.read(buckets.Waits())) != 0 || len(events) != 2 {
|
||||
t.Errorf("kept %v events %v", p.w.read(buckets.Waits()), events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnActingWaitOfANodeAwayStartsNoClockAndAnEntryWantedAlreadyWaitsNot(t *testing.T) {
|
||||
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.publishWants = func(string, any, string) error { t.Fatal("said"); return nil }
|
||||
p.graph = &graphKeeper{}
|
||||
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
|
||||
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": {Node: "ace", Module: "nats", Fp: "src2:a", Edges: []meshwants.Edge{
|
||||
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Step: "compose"}}}}
|
||||
n := assign.Nodes{Online: map[string]bool{}}
|
||||
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{})
|
||||
n.Online["ace"] = true
|
||||
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:a"}}})
|
||||
if len(p.w.read(buckets.Waits())) != 0 || strings.Contains(p.graph.computed.Entries["d1"].Fp, "x") {
|
||||
t.Errorf("kept %v", p.w.read(buckets.Waits()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsReadAndReleaseAnActingWait(t *testing.T) {
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
w := acting.Wait{Step: "compose", Node: "ace", Module: "nats", Part: "mesh-host", PartNode: "ace", Fp: "src2:new", Entry: "src2:a", Since: time.Now(), ClockFrom: time.Now(), Bound: "30m0s"}
|
||||
raw, _ := acting.Encode(w)
|
||||
s.st.put(waitsState, acting.Key("ace", "nats", "mesh-host"), raw)
|
||||
got, err := s.doWaits(map[string]any{"module": "nats"})
|
||||
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).BoundEnds == "" {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now"}, "x"); err == nil {
|
||||
t.Error("a release with no why is taken")
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now", "why": "the new engine is known good"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
|
||||
t.Fatalf("%v %v", err, acts)
|
||||
}
|
||||
raw, _, _, _ = s.st.get(waitsState, acting.Key("ace", "nats", "mesh-host"))
|
||||
if r, _ := acting.Decode(raw); !r.Released || !strings.Contains(r.By, "known good") {
|
||||
t.Errorf("wait = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §268, A-323, A-330: the assigner and the node-engine on a node are one step: the node-engine's own want waits for no
|
||||
// acting part, so it moves first, and the node's other wants, the assigner's own included, wait for the new node-engine; a want held
|
||||
// on a cycle no rule breaks is the finding acting-cycle (scope module, id <node>.<module>) naming each edge's step, cleared by the
|
||||
// next pass that finds it on none.
|
||||
func TestTheNodeEngineIsWantedFirstAndAWantHeldOnACycleIsAFinding(t *testing.T) {
|
||||
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Date(2026, 11, 13, 2, 0, 0, 0, time.UTC))
|
||||
p.publishWants = func(string, any, string) error { return nil }
|
||||
findings := map[string]mcseat.Finding{}
|
||||
p.findings = assign.Findings{
|
||||
Put: func(f mcseat.Finding) error { findings[f.Scope+"."+f.ID+"."+f.Kind] = f; return nil },
|
||||
Delete: func(key string) error { delete(findings, key); return nil },
|
||||
Keys: func() ([]string, error) {
|
||||
var keys []string
|
||||
for k := range findings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}}
|
||||
compose := meshwants.Edge{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:engine", Seat: "node-engine", Step: "compose"}
|
||||
place := meshwants.Edge{Kind: "acting", Node: "ace", Module: "assigner", Fp: "src2:assigner", Seat: "mesh-wants", Step: "place"}
|
||||
entries := map[string]meshwants.AssignmentEntry{
|
||||
"d1": {Node: "ace", Module: "mesh-host", Fp: "src2:engine", Edges: []meshwants.Edge{place}},
|
||||
"d2": {Node: "ace", Module: "assigner", Fp: "src2:assigner", Edges: []meshwants.Edge{compose}},
|
||||
"d3": {Node: "ace", Module: "app", Fp: "src2:app", Edges: []meshwants.Edge{compose, place}},
|
||||
}
|
||||
p.graph = &graphKeeper{}
|
||||
for d, e := range entries {
|
||||
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: e.Node, Module: e.Module, Digest: d})
|
||||
}
|
||||
p.graph.computed.Entries = entries
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {
|
||||
"mesh-host": {Fingerprint: "src2:old"}, "assigner": {Fingerprint: "src2:old"}}}}
|
||||
wanted := map[string]map[string]mwc.Want{}
|
||||
if got := p.actingOf("mesh-host", nil)("ace"); len(got) != 0 {
|
||||
t.Errorf("the node-engine's want is held by %+v, want by no acting edge", got)
|
||||
}
|
||||
if got := p.actingOf("assigner", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
|
||||
t.Errorf("the assigner's own want is held by %+v, want by the new node-engine", got)
|
||||
}
|
||||
p.actingWaits(context.Background(), n, wanted)
|
||||
kept := p.w.read(buckets.Waits())
|
||||
if kept[acting.Key("ace", "mesh-host", "assigner")] != nil || kept[acting.Key("ace", "assigner", "mesh-host")] == nil || kept[acting.Key("ace", "app", "mesh-host")] == nil {
|
||||
t.Errorf("the waits kept: %v", kept)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("the cycle the node-engine's want breaks is a finding: %v", findings)
|
||||
}
|
||||
// Two parts holding each other across compose and place, neither the node-engine: no rule breaks it.
|
||||
entries["d4"] = meshwants.AssignmentEntry{Node: "ace", Module: "x", Fp: "src2:x", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "y", Fp: "src2:y", Seat: "s-y", Step: "place"}}}
|
||||
entries["d5"] = meshwants.AssignmentEntry{Node: "ace", Module: "y", Fp: "src2:y", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "x", Fp: "src2:x", Seat: "s-x", Step: "compose"}}}
|
||||
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: "ace", Module: "x", Digest: "d4"},
|
||||
meshwants.NodeEntryRef{Node: "ace", Module: "y", Digest: "d5"})
|
||||
p.actingWaits(context.Background(), n, wanted)
|
||||
f, ok := findings["module.ace.x.acting-cycle"]
|
||||
if len(findings) != 2 || !ok || !strings.Contains(f.Summary, "ace/x -[place]-> ace/y -[compose]-> ace/x") {
|
||||
t.Fatalf("the findings of a cycle no rule breaks: %v", findings)
|
||||
}
|
||||
// x's new version runs: y no longer waits, so neither is held on a cycle, and both findings clear.
|
||||
n.Running["ace"]["x"] = nodeengine.Running{Fingerprint: "src2:x"}
|
||||
p.actingWaits(context.Background(), n, wanted)
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("the findings after the cycle broke: %v", findings)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// Which proposals the assigner asks of the operator is its setting ask-operator-for (design 47 G A-425, A-425a): a list of change
|
||||
// kinds, set at mesh level with mesh-config's config, its default take-off, remove-node and stops. A proposal none of whose changes
|
||||
// is listed is applied at once when its verdict passes, said in the log and by assignments-changed; one with a listed kind is
|
||||
// asked as before. The setting is read from the file the manifest declares with the content ${config:ask-operator-for}, which the
|
||||
// environment names (A-112), at every pass, and a change of it is said.
|
||||
|
||||
// KindStops is the change kind of a verdict whose answers say a node would stop running a module.
|
||||
const KindStops = "stops"
|
||||
|
||||
// defaultAskOperatorFor is the setting's default.
|
||||
var defaultAskOperatorFor = []string{mmac.OpTakeOff, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// askKinds are the kinds the setting may name.
|
||||
var askKinds = []string{mmac.OpPlace, mmac.OpTakeOff, mmac.OpPin, mmac.OpUnpin, mmac.OpHold, mmac.OpUnhold, mmac.OpDecide, mmac.OpConfig,
|
||||
mmac.OpAddNode, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// parseAskOperatorFor reads the setting's value: kinds separated by commas or spaces; empty is the default. A kind it does not
|
||||
// know is answered apart and named nowhere in the list.
|
||||
func parseAskOperatorFor(raw string) (kinds, unknown []string) {
|
||||
fields := strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
if len(fields) == 0 {
|
||||
return slices.Clone(defaultAskOperatorFor), nil
|
||||
}
|
||||
for _, f := range fields {
|
||||
f = strings.ToLower(strings.TrimSpace(f))
|
||||
switch {
|
||||
case !slices.Contains(askKinds, f):
|
||||
unknown = append(unknown, f)
|
||||
case !slices.Contains(kinds, f):
|
||||
kinds = append(kinds, f)
|
||||
}
|
||||
}
|
||||
slices.Sort(kinds)
|
||||
return kinds, unknown
|
||||
}
|
||||
|
||||
// readAskOperatorFor reads the setting's file: the default for no path or no file.
|
||||
func readAskOperatorFor(path string) (kinds, unknown []string, err error) {
|
||||
if path == "" {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
k, u := parseAskOperatorFor(string(raw))
|
||||
return k, u, nil
|
||||
}
|
||||
|
||||
// askOperatorFor is the setting as this pass read it: the file read again each pass, a change said once, a file that cannot be
|
||||
// read keeping what was read last (the default before any).
|
||||
func (p *passer) askOperatorFor() []string {
|
||||
kinds, unknown, err := readAskOperatorFor(p.askForFile)
|
||||
if err != nil {
|
||||
if p.askFor == nil {
|
||||
p.askFor = slices.Clone(defaultAskOperatorFor)
|
||||
}
|
||||
p.logf("the setting ask-operator-for cannot be read from %s, so %s stands: %v", p.askForFile, strings.Join(p.askFor, ", "), err)
|
||||
return p.askFor
|
||||
}
|
||||
if !slices.Equal(kinds, p.askFor) {
|
||||
words := "the setting ask-operator-for is " + strings.Join(kinds, ", ") + ": a proposal with a change of another kind only is applied with no ask"
|
||||
if len(unknown) > 0 {
|
||||
words += "; kinds it names that are none are left out: " + strings.Join(unknown, ", ")
|
||||
}
|
||||
p.logf("%s", words)
|
||||
p.askFor = kinds
|
||||
}
|
||||
return p.askFor
|
||||
}
|
||||
|
||||
// asksFor says whether a proposal of these change kinds is asked of the operator.
|
||||
func (p *passer) asksFor(kinds []string) bool {
|
||||
set := p.askOperatorFor()
|
||||
for _, k := range kinds {
|
||||
if slices.Contains(set, k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// proposalKinds are the change kinds of a proposal whose verdict is v: each change's op, and stops when an answer says a node
|
||||
// would stop running a module.
|
||||
func proposalKinds(prop mma.Proposal, v mma.Verdict) []string {
|
||||
var out []string
|
||||
for _, c := range prop.Changes {
|
||||
if c.Op != "" && !slices.Contains(out, c.Op) {
|
||||
out = append(out, c.Op)
|
||||
}
|
||||
}
|
||||
if len(judge.Stops(v)) > 0 {
|
||||
out = append(out, KindStops)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks
|
||||
// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps
|
||||
// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first.
|
||||
|
||||
const (
|
||||
asksTool = "assigner_asks"
|
||||
askDropTool = "assigner_ask_drop"
|
||||
)
|
||||
|
||||
// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total).
|
||||
const asksAnswerMost = 768 << 10
|
||||
|
||||
// takenReader is what assigner_asks reads of the state taken (stdio.KeptState).
|
||||
type takenReader interface {
|
||||
Keys() ([]string, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
}
|
||||
|
||||
// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.<ask>) and the drops of
|
||||
// asks not pulled yet (dropped.<ask>), each list cut at the answer's bound with capped and total.
|
||||
func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) {
|
||||
w, err := waiting(meshwants.Seat)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err)
|
||||
}
|
||||
keys, err := taken.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
// An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a
|
||||
// nak (H-A-410's addenda): the bus says what it knows, the state taken the rest.
|
||||
size := 0
|
||||
waits := make([]map[string]any, 0, len(w.Asks))
|
||||
for _, a := range w.Asks {
|
||||
size += len(a.ID) + len(a.Module) + 64
|
||||
state := "waiting"
|
||||
if a.Again {
|
||||
state = "waiting again"
|
||||
if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) {
|
||||
state = "held"
|
||||
}
|
||||
}
|
||||
waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state})
|
||||
}
|
||||
takenIDs, dropped := []string{}, []map[string]any{}
|
||||
total, capped := 0, w.Capped
|
||||
for _, k := range keys {
|
||||
if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok {
|
||||
total++
|
||||
if size += len(id) + 4; size > asksAnswerMost {
|
||||
capped = true
|
||||
continue
|
||||
}
|
||||
takenIDs = append(takenIDs, id)
|
||||
continue
|
||||
}
|
||||
id, ok := strings.CutPrefix(k, droppedPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
total++
|
||||
e, err := taken.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
var d askDrop
|
||||
if json.Unmarshal(e.Value, &d) != nil {
|
||||
continue
|
||||
}
|
||||
if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost {
|
||||
capped = true
|
||||
continue
|
||||
}
|
||||
dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At})
|
||||
}
|
||||
return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil
|
||||
}
|
||||
|
||||
// asksTools are the module's tools of the seat's work queue.
|
||||
func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " +
|
||||
"that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " +
|
||||
"cut at 768 KB with capped and total. Read-only.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return asksOf(waiting, taken)
|
||||
}},
|
||||
{Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " +
|
||||
"first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " +
|
||||
"its step could not run naming the drop and who asked it.",
|
||||
Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why"))
|
||||
if id == "" || why == "" {
|
||||
return nil, errors.New("name the ask and why it is ended")
|
||||
}
|
||||
by := stdio.CallerOf(args)
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err)
|
||||
}
|
||||
out["hand-act"] = taken.Log
|
||||
return out, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat
|
||||
// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process.
|
||||
const keepNowTool = "assigner_keep_now"
|
||||
|
||||
func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " +
|
||||
"hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " +
|
||||
"the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).",
|
||||
Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
why := strings.TrimSpace(arg(args, "why"))
|
||||
if why == "" {
|
||||
return nil, errors.New("say why the keeping runs now")
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err)
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err)
|
||||
}
|
||||
out["hand-act"] = taken.Log
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
|
||||
if v, _, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
|
||||
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -97,7 +100,7 @@ func (r *releasedVersions) put(data []byte) error {
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for _, a := range e.Artifacts {
|
||||
parts[e.Module+":"+a.Artifact] = a.Digest
|
||||
parts[partRef(e.Module, a.Artifact)] = a.Digest
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -116,6 +119,48 @@ func (r *releasedVersions) put(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// withParts is the version of module whose release named exactly parts (`<module>:<artifact>` -> digest), the newest such when
|
||||
// more than one did; false when none did or parts is empty (design 47 G A-421).
|
||||
func (r *releasedVersions) withParts(module string, parts map[string]string) (assign.KeptBuild, bool) {
|
||||
if len(parts) == 0 {
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out assign.KeptBuild
|
||||
found := false
|
||||
for key, k := range r.seen {
|
||||
if m, _, ok := strings.Cut(key, "@"); !ok || m != module || !maps.Equal(k.Parts, parts) {
|
||||
continue
|
||||
}
|
||||
if !found || compareVersion(k.Version, out.Version) > 0 {
|
||||
out, found = k, true
|
||||
}
|
||||
}
|
||||
return out, found
|
||||
}
|
||||
|
||||
// madeOf is what module's done build at fp made, by `<module>:<artifact>`, as its build.ended names it; none for a build not heard
|
||||
// or not done (design 47 G A-421).
|
||||
func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
e, ok := b.endOf(module, fp)
|
||||
if !ok || e.Outcome != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for name, digest := range e.Made {
|
||||
parts[module+":"+name] = digest
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// releasedOf is the version released of module's done build at fp, the release whose parts are what its build.ended made (design
|
||||
// 47 G A-421); false when no such release is heard.
|
||||
func (w *world) releasedOf(module, fp string) (string, bool) {
|
||||
k, ok := w.released.withParts(module, w.ends.madeOf(module, fp))
|
||||
return k.Version, ok && k.Version != ""
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
@@ -143,7 +188,7 @@ func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wa
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf)
|
||||
}, logf, process.FromFirst()) // read whole from its first message whether the bus keeps it last per subject or by age (A-181, A-233)
|
||||
if err != nil {
|
||||
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
|
||||
return
|
||||
@@ -155,16 +200,29 @@ func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wa
|
||||
}()
|
||||
}
|
||||
|
||||
// hearTaken registers the hub's taker of the indexer's take-in.taken, each waking the pass (orderwaits.go, §298, A-260); before
|
||||
// the hub starts.
|
||||
func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf func(string, ...any)) error {
|
||||
return hub.Handle(takenSubject, func(m jetstream.Msg) {
|
||||
if err := taken.put(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
// compareVersion orders two MAJOR.MINOR.PATCH versions by their numbers; a part that is no number counts as 0.
|
||||
func compareVersion(a, b string) int {
|
||||
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; i < 3; i++ {
|
||||
var x, y int
|
||||
if i < len(pa) {
|
||||
x, _ = strconv.Atoi(pa[i])
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
if i < len(pb) {
|
||||
y, _ = strconv.Atoi(pb[i])
|
||||
}
|
||||
wake()
|
||||
})
|
||||
if x != y {
|
||||
return x - y
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// partRef is a release's artifact as a part's key, `<module>:<artifact>`: the contract's VersionArtifact names the artifact
|
||||
// reference whole (an ArtifactRef), as the catalog's version-released says it; a bare name is taken as the module's (A-96).
|
||||
func partRef(module, artifact string) string {
|
||||
if strings.Contains(artifact, ":") {
|
||||
return artifact
|
||||
}
|
||||
return module + ":" + artifact
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
@@ -54,11 +56,12 @@ func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §379, A-96: the trunk's entry takes its version and parts from the version-released of the kept assignment graph
|
||||
// revision's source; a kept build from its end; no revision kept, or another fingerprint, names none.
|
||||
func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *testing.T) {
|
||||
// Design 47 G §379, A-96, A-421; H §764, H-A-48: the trunk's entry, which the catalog stamps with no source, takes its version and
|
||||
// parts from the version-released whose parts are what its build.ended made; a kept build from its end; no revision kept, another
|
||||
// fingerprint and a build whose parts no release names name none, and an unheard release is said once.
|
||||
func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsBuildsParts(t *testing.T) {
|
||||
w := newWorld()
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -66,45 +69,44 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes
|
||||
if err := w.ends.put(endRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
|
||||
var logs []string
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}, logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
named := p.namedBuild("mail", "t")
|
||||
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" || k.KeptOn != "" {
|
||||
t.Errorf("trunk entry = %+v, %v", k, ok)
|
||||
if _, ok := named("t"); ok {
|
||||
t.Error("a trunk entry whose build.ended is not heard names a version")
|
||||
}
|
||||
_, _ = named("t")
|
||||
if k, ok := named("k"); !ok || k.KeptOn != "g14" || k.Version != "" {
|
||||
t.Errorf("kept build = %+v, %v", k, ok)
|
||||
}
|
||||
if _, ok := named("other"); ok {
|
||||
t.Error("another fingerprint names a release")
|
||||
}
|
||||
p.graph.source = 8
|
||||
if _, ok := named("t"); ok {
|
||||
t.Error("a source with no release names a version")
|
||||
// A build held back at an older fingerprint names the release whose parts are exactly what it made.
|
||||
heldRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "old", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(heldRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k, ok := named("old"); !ok || k.Version != "1.2.3" {
|
||||
t.Errorf("a held build whose parts were released = %+v, %v; want 1.2.3", k, ok)
|
||||
}
|
||||
// The entry's module value carries no source stamp: the catalog writes none (H-A-48).
|
||||
if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{}}); v.Fp2 != "t" {
|
||||
t.Errorf("the entry's module value is %+v", v)
|
||||
}
|
||||
made, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "t", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(made); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.graph.source = 8 // a later merge that did not rebuild mail
|
||||
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" {
|
||||
t.Errorf("the trunk entry whose build made the released digests: %+v, %v", k, ok)
|
||||
}
|
||||
if len(logs) != 1 || !strings.Contains(logs[0], "the release of mail at fingerprint t is not heard: no version-released of it names the parts its build.ended made") {
|
||||
t.Errorf("logs = %q, want the unheard release said once, naming the module and the fingerprint", logs)
|
||||
}
|
||||
p.graph.ok = false
|
||||
if _, ok := p.namedBuild("mail", "t")("t"); ok {
|
||||
t.Error("no revision kept names a release")
|
||||
}
|
||||
}
|
||||
|
||||
// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed.
|
||||
func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) {
|
||||
w := newWorld()
|
||||
say := func(e builds.BuildEnded) {
|
||||
raw, _ := json.Marshal(e)
|
||||
if err := w.ends.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if failed, _ := w.buildStanding("web", "fp1"); failed {
|
||||
t.Error("a build nothing was heard of failed")
|
||||
}
|
||||
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"})
|
||||
if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" {
|
||||
t.Errorf("failed end: failed %v why %q", failed, why)
|
||||
}
|
||||
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone})
|
||||
if failed, _ := w.buildStanding("web", "fp1"); failed {
|
||||
t.Error("a later done end leaves the build failed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,18 +28,25 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
if p.ends.following() {
|
||||
branch = p.ends.branchOf(module, placed[module])
|
||||
}
|
||||
fp := p.w.trunkOf(branch)[module].Fp2
|
||||
entry := p.w.trunkOf(branch)[module]
|
||||
fp := entry.Fp2
|
||||
if fp == "" {
|
||||
p.skippedAdd(node, module, "", "the branch "+branch+" names no fingerprint of it")
|
||||
continue
|
||||
}
|
||||
k, ok := p.namedBuild(module, fp)(fp)
|
||||
if !ok {
|
||||
p.skippedAdd(node, module, fp, "no version-released of it is heard for that fingerprint, so the candidate carries no version and no parts")
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]mma.AddedModule{}
|
||||
}
|
||||
out[module] = mma.AddedModule{Version: k.Version, Parts: k.Parts}
|
||||
added := mma.AddedModule{Version: k.Version, Parts: k.Parts}
|
||||
if k.Version != "" {
|
||||
added.Parts = nil // a released build is named by its version alone (design 47 G A-430, A-430a)
|
||||
}
|
||||
out[module] = added
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -48,3 +55,18 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
|
||||
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
|
||||
}
|
||||
|
||||
// skippedAdd says once per node, module and fingerprint why a module the candidate adds is not listed in `added`.
|
||||
func (p *passer) skippedAdd(node, module, fp, why string) {
|
||||
key := node + "/" + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the candidate adds %s to %s with no parts: %s (fingerprint %q)", module, node, why, fp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,21 +20,25 @@ func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
|
||||
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
|
||||
}
|
||||
w.first = "main"
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(rel); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-mail", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
end, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: "fp-db", Made: map[string]string{"program": "sha256:3"}, KeptOn: "g14"})
|
||||
if err := w.ends.put(end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 9}} // a later merge: the release is matched by the parts its build made (A-421, H-A-48)
|
||||
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
|
||||
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
|
||||
c := p.candidateOf("p1", "ace", laid, current)
|
||||
want := map[string]mma.AddedModule{
|
||||
"mail": {Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}},
|
||||
"mail": {Version: "1.2.3"}, // a released build is named by its version alone (A-430a)
|
||||
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
|
||||
}
|
||||
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
|
||||
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
|
||||
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
|
||||
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
|
||||
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
|
||||
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
|
||||
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
|
||||
|
||||
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
|
||||
const waitOnBuildNode = "build node"
|
||||
|
||||
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
|
||||
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
|
||||
|
||||
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
|
||||
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
|
||||
return func(ask, why string) error {
|
||||
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, said, ok := buscalls.Unwrap(raw); !ok {
|
||||
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
|
||||
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
|
||||
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
|
||||
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
|
||||
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
|
||||
if run.Started != nil {
|
||||
w.Since = *run.Started
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
|
||||
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
|
||||
if r.say == nil {
|
||||
return
|
||||
}
|
||||
for _, c := range readers {
|
||||
w := waitOf(run, c)
|
||||
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
|
||||
if how != "" {
|
||||
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
|
||||
}
|
||||
if err := r.say(event, w, id); err != nil {
|
||||
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
|
||||
func (r *compositionRuns) untaken(key, ask string) bool {
|
||||
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
|
||||
return err == nil && standing && held == ask
|
||||
}
|
||||
|
||||
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
|
||||
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(boundKey)
|
||||
if err != nil || raw == nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(raw, &v)
|
||||
return v.Ask, rev, true, nil
|
||||
}
|
||||
|
||||
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
|
||||
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
|
||||
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
|
||||
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
|
||||
rev, err := r.kv.arm(prefix+key, ask, ttl)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok && run.Ended == nil && run.Ask == ask {
|
||||
return true, nil
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
|
||||
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
|
||||
// ended, which moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
|
||||
key, run, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
|
||||
}
|
||||
if !armed {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.untaken(key, e.Ask) {
|
||||
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
r.sayWaits(run, run.Readers, "ran")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
|
||||
// late one moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
|
||||
key, _, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
armed := false
|
||||
if key != "" {
|
||||
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !armed {
|
||||
r.late("step-progress", e.Asker, e.Ask)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
|
||||
func (r *compositionRuns) late(signal, asker, ask string) {
|
||||
if asker == runAsker && ask != "" {
|
||||
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
|
||||
}
|
||||
}
|
||||
|
||||
// runOfAsk is the running run of an ask the assigner put, or none.
|
||||
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
|
||||
if asker != runAsker || ask == "" {
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(ask)
|
||||
if err != nil || key == "" || run.Ended != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
return key, run, nil
|
||||
}
|
||||
|
||||
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
|
||||
// ask and the bound. A run that ended already, or a key of none, takes nothing.
|
||||
func (r *compositionRuns) expired(boundKey string) {
|
||||
key, bound, setting := "", time.Duration(0), ""
|
||||
switch {
|
||||
case strings.HasPrefix(boundKey, runBoundPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
|
||||
case strings.HasPrefix(boundKey, runUntakenPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
|
||||
default:
|
||||
return
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
return
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
|
||||
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
|
||||
// since, ends nothing.
|
||||
other := runUntakenPrefix
|
||||
if setting == "run-untaken-bound" {
|
||||
other = runBoundPrefix
|
||||
}
|
||||
_, _, armed, err := r.boundOf(boundKey)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
|
||||
return
|
||||
}
|
||||
held, _, standing, err := r.boundOf(other + key)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
|
||||
return
|
||||
}
|
||||
if armed || (standing && held == run.Ask) {
|
||||
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
|
||||
return
|
||||
}
|
||||
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
if setting == "run-untaken-bound" {
|
||||
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
}
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
|
||||
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
|
||||
r.logf("the composition run %s past its bound is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
|
||||
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
|
||||
}
|
||||
|
||||
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
|
||||
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
|
||||
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
|
||||
}
|
||||
|
||||
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
|
||||
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
|
||||
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
|
||||
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
|
||||
if moved := r.moved.Load(); moved != nil && move {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
|
||||
(*moved)(run)
|
||||
}
|
||||
}
|
||||
ended, err := r.end(key, ask, outcome, said, verdict)
|
||||
if err != nil || !ended {
|
||||
return ended, err
|
||||
}
|
||||
if untaken {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok {
|
||||
r.sayWaits(run, run.Readers, "superseded")
|
||||
}
|
||||
}
|
||||
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
|
||||
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
|
||||
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
|
||||
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []openReader
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range run.Readers {
|
||||
if c.Ended != nil && !c.Rerun {
|
||||
continue // it ended passed already: only a reader marked rerun says its end again
|
||||
}
|
||||
if !seen[c.ID] {
|
||||
seen[c.ID] = true
|
||||
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
|
||||
type openReader struct {
|
||||
meshwants.CompositionReader
|
||||
ask string
|
||||
}
|
||||
|
||||
// keysOf are the keys of the candidate's runs that the reader reads.
|
||||
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
prefix := keysPart(candidate) + "."
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out) // by machine, as a step names its machines
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
|
||||
// handed to the runs; one that cannot be written is given again by the bus.
|
||||
func hearRunBounds(hub interface {
|
||||
Handle(string, func(jetstream.Msg)) error
|
||||
}, r *compositionRuns, logf func(string, ...any)) error {
|
||||
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardAskTaken(e), logf)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.StepProgress
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardProgress(e), logf)
|
||||
})
|
||||
}
|
||||
|
||||
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
|
||||
if err != nil {
|
||||
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
|
||||
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
|
||||
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil {
|
||||
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
|
||||
return
|
||||
}
|
||||
for _, reader := range readers {
|
||||
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
|
||||
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = reader.ask
|
||||
}
|
||||
how := c.howOf(reader.Step, true)
|
||||
if how == nil {
|
||||
c.logf("the step %s of %s is not taken up again: it is no composition or validate step (%q)", reader.ID, reader.Module, reader.Step)
|
||||
continue
|
||||
}
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), how, again)
|
||||
}
|
||||
}
|
||||
@@ -3,22 +3,24 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
|
||||
// asked of the machine's node-engine by the seat's verb compose-candidate.
|
||||
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
|
||||
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
|
||||
|
||||
// composeCandidateKey is the seat verb's call key, asked of one node.
|
||||
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
|
||||
@@ -28,11 +30,11 @@ const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
|
||||
func candidateKeyOf(run meshwants.StepRun) (string, error) {
|
||||
switch {
|
||||
case run.Run != "":
|
||||
return "run-" + run.Run, nil
|
||||
return nec.RunKey(run.Run)
|
||||
case run.Branch == "" || run.Revision <= 0:
|
||||
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
|
||||
}
|
||||
return keys.EscapeName(run.Branch) + "." + strconv.FormatInt(run.Revision, 10), nil
|
||||
return nec.RevisionKey(run.Branch, run.Revision) // built by the seat's client, never by hand (A-226b)
|
||||
}
|
||||
|
||||
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
|
||||
@@ -72,62 +74,64 @@ type nodeInput struct {
|
||||
// compositionInputs asks the machines of a step for their inputs.
|
||||
type compositionInputs struct {
|
||||
machines func(module, branch string) []string
|
||||
// call asks one node's seat verb (process.Conn.CallKeptOn).
|
||||
call func(node, key string, body any) (json.RawMessage, error)
|
||||
// callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to
|
||||
// one node as `seat:<seat>.<verb>@<node>`, carrying the ask's id and stopping at its bound, and answers the verb's answer with
|
||||
// the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the
|
||||
// body's `call`.
|
||||
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
// bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775).
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
|
||||
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
|
||||
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
|
||||
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
|
||||
key, err := candidateKeyOf(run)
|
||||
// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no
|
||||
// input, naming every call.
|
||||
const (
|
||||
composeAskBoundSetting = "compose-ask-bound"
|
||||
defaultComposeAskBound = 10 * time.Second
|
||||
)
|
||||
|
||||
// composeAskBound reads the setting compose-ask-bound (§781).
|
||||
func composeAskBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
nodes := c.machines(run.Module, run.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
|
||||
}
|
||||
bound := c.bound
|
||||
if bound <= 0 {
|
||||
bound = nec.AnswerBound
|
||||
}
|
||||
out := make([]nodeInput, len(nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
if away != nil && away(node) {
|
||||
out[i] = nodeInput{Node: node, Away: true}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out[i] = c.one(ctx, bound, node, key)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// one asks one node for its input.
|
||||
func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput {
|
||||
// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per
|
||||
// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id.
|
||||
func composeAskID(stepAsk, node string) string { return stepAsk + "." + node }
|
||||
|
||||
// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step
|
||||
// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until
|
||||
// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set.
|
||||
func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput {
|
||||
bound := c.bound
|
||||
if bound <= 0 {
|
||||
bound = defaultComposeAskBound
|
||||
}
|
||||
in := nodeInput{Node: node}
|
||||
at, cancel := context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
type reply struct {
|
||||
raw json.RawMessage
|
||||
err error
|
||||
}
|
||||
got := make(chan reply, 1)
|
||||
go func() {
|
||||
raw, err := c.call(node, composeCandidateKey, ne.ComposeCandidateInput{Candidate: key, Input: true})
|
||||
// The input is any size, so the verb sends it in chunks on the call token chosen here, and the file is the input (A-237b, A-237c).
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
got <- reply{nil, err}
|
||||
return
|
||||
}
|
||||
raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node,
|
||||
ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])})
|
||||
got <- reply{raw, err}
|
||||
}()
|
||||
var raw json.RawMessage
|
||||
select {
|
||||
case <-at.Done():
|
||||
in.Why = "no answer within its bound: " + at.Err().Error()
|
||||
case <-ctx.Done():
|
||||
in.Why = "its step's wait ended: " + context.Cause(ctx).Error()
|
||||
return in
|
||||
case r := <-got:
|
||||
if r.err != nil {
|
||||
@@ -148,17 +152,36 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node,
|
||||
case a.Verdict == nec.Unready:
|
||||
in.Why = "unready: " + a.Refused
|
||||
return in
|
||||
case len(a.Input) == 0:
|
||||
case a.Chunks == 0 && len(a.Input) == 0:
|
||||
in.Why = "the engine answered no composition input"
|
||||
if a.Refused != "" {
|
||||
in.Why += ": " + a.Refused
|
||||
}
|
||||
return in
|
||||
}
|
||||
body, err := json.Marshal(a.Input)
|
||||
if err != nil {
|
||||
in.Why = "its composition input cannot be encoded again: " + err.Error()
|
||||
return in
|
||||
var body []byte
|
||||
if a.Chunks > 0 {
|
||||
// The composition input is the file the chunks carried, the engine's own bytes (numbers as written).
|
||||
var whole struct {
|
||||
File string `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &whole); err != nil {
|
||||
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
|
||||
return in
|
||||
}
|
||||
file, err := base64.StdEncoding.DecodeString(whole.File)
|
||||
if err != nil || len(file) == 0 {
|
||||
in.Why = fmt.Sprintf("the engine named %d chunks of composition input and the file received is empty or not base64: %v", a.Chunks, err)
|
||||
return in
|
||||
}
|
||||
body = file
|
||||
} else {
|
||||
// An answer with the input inline (an engine before A-237b, read until the checker is removed, A-237c, G-6 §321): the object, as it was.
|
||||
var err error
|
||||
if body, err = json.Marshal(a.Input); err != nil {
|
||||
in.Why = "its composition input cannot be encoded again: " + err.Error()
|
||||
return in
|
||||
}
|
||||
}
|
||||
in.Body = body
|
||||
return in
|
||||
|
||||
+81
-9
@@ -2,13 +2,17 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
@@ -21,12 +25,21 @@ func TestTheCandidateKeyOfAStep(t *testing.T) {
|
||||
}{
|
||||
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
|
||||
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 362}, want: "main.362"},
|
||||
{run: meshwants.StepRun{Branch: "main"}, fail: true},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 100000000000}, fail: true}, // a number the client's key would read as a heads hash
|
||||
{run: meshwants.StepRun{Run: "a.b"}, fail: true}, // a run id the client refuses
|
||||
} {
|
||||
got, err := candidateKeyOf(c.run)
|
||||
if (err != nil) != c.fail || got != c.want {
|
||||
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
|
||||
}
|
||||
// A-226b: the node-engine reads every key the step names with the client's one parser.
|
||||
if !c.fail {
|
||||
if _, err := nec.ParseCandidateKey(got); err != nil {
|
||||
t.Errorf("%+v: the key %q is refused by the seat's parser: %v", c.run, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,13 +69,15 @@ func TestTheMachinesOfAStep(t *testing.T) {
|
||||
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
|
||||
// silent or unready node says why, and a machine away is not asked.
|
||||
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
// gather asks the machines side by side, so the record of what each was asked is guarded.
|
||||
// the machines are asked side by side, so the record of what each was asked is guarded.
|
||||
var mu sync.Mutex
|
||||
asked := map[string]string{}
|
||||
call := func(node, key string, body any) (json.RawMessage, error) {
|
||||
asks := map[string]process.KeptAsk{}
|
||||
call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
mu.Lock()
|
||||
asked[node] = key + " " + string(raw)
|
||||
asked[node], asks[node] = key+" "+string(raw), ask
|
||||
mu.Unlock()
|
||||
switch node {
|
||||
case "g14":
|
||||
@@ -72,25 +87,82 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
}
|
||||
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, call: call, bound: time.Second}
|
||||
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
if err != nil || len(got) != 4 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if want := `seat:node-engine.compose-candidate {"candidate":"run-r1","input":true}`; asked["ace"] != want {
|
||||
t.Errorf("asked %q, want %q", asked["ace"], want)
|
||||
// With a call token of 128 random bits (32 hex digits), the verb being marked chunks.
|
||||
if k, body, _ := strings.Cut(asked["ace"], " "); k != "seat:node-engine.compose-candidate" ||
|
||||
!regexp.MustCompile(`^\{"call":"[0-9a-f]{32}","candidate":"run-r1","input":true\}$`).MatchString(body) {
|
||||
t.Errorf("asked %q", asked["ace"])
|
||||
}
|
||||
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
|
||||
t.Errorf("ace: %+v", got[0])
|
||||
}
|
||||
// Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound.
|
||||
if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" {
|
||||
t.Errorf("the kept asks %+v", asks)
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
|
||||
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
|
||||
}
|
||||
if _, was := asked["shanks"]; was || !got[3].Away {
|
||||
t.Errorf("the machine away is asked or not said away: %+v", got[3])
|
||||
}
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, call: call}
|
||||
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
|
||||
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
t.Error("a step with no machine is gathered")
|
||||
}
|
||||
}
|
||||
|
||||
// A-237b: the composition input is any size, so a node names the chunks it sent and the assigner reads the file the chunked call
|
||||
// received as the input, its numbers as written; a node that names chunks and sends no file is said so.
|
||||
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
|
||||
file := base64.StdEncoding.EncodeToString([]byte(input))
|
||||
call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@")
|
||||
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
|
||||
if node == "g14" {
|
||||
return json.RawMessage(answer + `}`), nil // names chunks, none received
|
||||
}
|
||||
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if got[0].Why != "" || string(got[0].Body) != input {
|
||||
t.Errorf("ace: why %q, %d bytes, want the file received (%d bytes) as the input", got[0].Why, len(got[0].Body), len(input))
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "chunks") || got[1].Body != nil {
|
||||
t.Errorf("g14: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
|
||||
// is not asked; no machine is an error.
|
||||
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
|
||||
key, err := candidateKeyOf(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := c.machines(run.Module, run.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return nil, errors.New("no machine")
|
||||
}
|
||||
out := make([]nodeInput, len(nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
if away != nil && away(node) {
|
||||
out[i] = nodeInput{Node: node, Away: true}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
|
||||
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
|
||||
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
|
||||
// could not run.
|
||||
|
||||
const (
|
||||
// readsBound is how long a read is made again before the step could not run.
|
||||
readsBound = 3 * time.Minute
|
||||
// readsFirstWait and readsMaxWait bound the wait between two tries.
|
||||
readsFirstWait = time.Second
|
||||
readsMaxWait = 15 * time.Second
|
||||
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
|
||||
readsKept = 8
|
||||
)
|
||||
|
||||
// readKey names one revision of one watched branch.
|
||||
type readKey struct {
|
||||
branch string
|
||||
revision int
|
||||
}
|
||||
|
||||
// readSlot is one revision's read: done closes when half or err is set.
|
||||
type readSlot struct {
|
||||
done chan struct{}
|
||||
half mcc.Half
|
||||
err error
|
||||
}
|
||||
|
||||
// compositionReads shares the reads of composition steps.
|
||||
type compositionReads struct {
|
||||
mu sync.Mutex
|
||||
revs map[readKey]*readSlot
|
||||
order []readKey
|
||||
seats map[string]json.RawMessage // by seat and digest: the definition, raw
|
||||
}
|
||||
|
||||
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
|
||||
func transient(err error, said string) bool {
|
||||
text := said
|
||||
if err != nil {
|
||||
text = err.Error() + " " + said
|
||||
}
|
||||
text = strings.ToLower(text)
|
||||
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
|
||||
if strings.Contains(text, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
|
||||
// a good answer, or the last fault.
|
||||
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
|
||||
start := r.now()
|
||||
wait := readsFirstWait
|
||||
for try := 1; ; try++ {
|
||||
raw, err := r.call(key, body)
|
||||
var inner json.RawMessage
|
||||
var said string
|
||||
if err == nil {
|
||||
var ok bool
|
||||
if inner, said, ok = buscalls.Unwrap(raw); ok {
|
||||
return inner, "", nil
|
||||
}
|
||||
}
|
||||
if !transient(err, said) {
|
||||
return nil, said, err
|
||||
}
|
||||
if r.now().Sub(start)+wait > readsBound {
|
||||
return nil, said, err
|
||||
}
|
||||
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
|
||||
if serr := r.waitFor(ctx, wait); serr != nil {
|
||||
return nil, said, serr
|
||||
}
|
||||
if wait *= 2; wait > readsMaxWait {
|
||||
wait = readsMaxWait
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor waits d, or until ctx ends.
|
||||
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
|
||||
if r.sleep != nil {
|
||||
return r.sleep(ctx, d)
|
||||
}
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
|
||||
// failed is not kept, so the next step reads again.
|
||||
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
|
||||
c := r.reads
|
||||
c.mu.Lock()
|
||||
if c.revs == nil {
|
||||
c.revs = map[readKey]*readSlot{}
|
||||
}
|
||||
if s, ok := c.revs[key]; ok {
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
if s.err == nil {
|
||||
return s.half, nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return mcc.Half{}, ctx.Err()
|
||||
}
|
||||
// The read this step waited for failed: read it here (the failed slot is already dropped).
|
||||
return r.half(ctx, key, read)
|
||||
}
|
||||
s := &readSlot{done: make(chan struct{})}
|
||||
c.revs[key] = s
|
||||
c.order = append(c.order, key)
|
||||
for len(c.order) > readsKept {
|
||||
delete(c.revs, c.order[0])
|
||||
c.order = c.order[1:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
s.half, s.err = read()
|
||||
if s.err != nil {
|
||||
c.mu.Lock()
|
||||
if c.revs[key] == s {
|
||||
delete(c.revs, key)
|
||||
}
|
||||
for i, k := range c.order {
|
||||
if k == key {
|
||||
c.order = append(c.order[:i], c.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
close(s.done)
|
||||
return s.half, s.err
|
||||
}
|
||||
|
||||
// seatKept is the definition kept of a seat at a digest.
|
||||
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
d, ok := c.seats[seat+"@"+digest]
|
||||
return d, ok
|
||||
}
|
||||
|
||||
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.seats == nil {
|
||||
c.seats = map[string]json.RawMessage{}
|
||||
}
|
||||
c.seats[seat+"@"+digest] = def
|
||||
}
|
||||
|
||||
func unreadErr(what string, err error, said string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", what, said)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,17 +4,25 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -30,13 +38,23 @@ type runStand struct {
|
||||
requests map[string]composer.Request // by node
|
||||
entries []string // the fingerprints `entry` was asked
|
||||
r *compositionRun
|
||||
runs *compositionRuns
|
||||
puts int // the run asks put
|
||||
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
|
||||
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
|
||||
noDefinitions bool
|
||||
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
|
||||
unjudged string
|
||||
listFault error
|
||||
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
|
||||
verdict func(node string) (composer.Machine, bool)
|
||||
}
|
||||
|
||||
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
|
||||
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
|
||||
engine := func(fp, commit string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
|
||||
Manifest: map[string]any{"module": "engine"}}
|
||||
Manifest: map[string]any{"module": "engine"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25-" + fp, "git.novox.be/novox/x/go": ""}}
|
||||
}
|
||||
revisionCommit := commitE1
|
||||
if revisionEngine != "src2:e1" {
|
||||
@@ -46,7 +64,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
|
||||
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
|
||||
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
|
||||
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
|
||||
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
|
||||
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
@@ -57,8 +76,22 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
if in.Revision != 7 || in.Branch != "main" {
|
||||
return nil, errors.New("asked another revision")
|
||||
}
|
||||
return json.Marshal(pages[in.After])
|
||||
page := pages[in.After]
|
||||
if s.noDefinitions && page.Revision != nil {
|
||||
rev := *page.Revision
|
||||
rev.Definitions = nil
|
||||
page.Revision = &rev
|
||||
}
|
||||
return json.Marshal(page)
|
||||
case seatDefinitionsKey:
|
||||
var sin meshcatalog.SeatDefinitionsInput
|
||||
_ = json.Unmarshal(raw, &sin)
|
||||
if sin.Seat == "" {
|
||||
if s.listFault != nil {
|
||||
return nil, s.listFault
|
||||
}
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Seats: s.trunkSeats})
|
||||
}
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
|
||||
case entryVerbKey:
|
||||
var in meshcatalog.EntryInput
|
||||
@@ -71,17 +104,17 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
engines := func(node, _ string, _ any) (json.RawMessage, error) {
|
||||
engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
|
||||
away := &awayNodes{}
|
||||
away.set("shanks", true)
|
||||
away := func(machine string) bool { return machine == "shanks" }
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
ends := &runEnds{}
|
||||
s.r = &compositionRun{
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, call: engines, bound: time.Second},
|
||||
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
|
||||
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
|
||||
s.r = &compositionRun{runs: s.runs,
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
|
||||
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
|
||||
trunk: func() (mcc.Half, bool) {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
|
||||
},
|
||||
@@ -103,11 +136,27 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
s.asks[node], s.requests[node] = a, req
|
||||
s.mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
if node == "g14" {
|
||||
override, set := composer.Machine{}, false
|
||||
if s.verdict != nil {
|
||||
override, set = s.verdict(node)
|
||||
}
|
||||
if set {
|
||||
m = override
|
||||
} else if node == s.unjudged {
|
||||
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
|
||||
} else if node == "g14" {
|
||||
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
|
||||
}
|
||||
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
|
||||
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
|
||||
s.mu.Lock()
|
||||
s.puts++
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
|
||||
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -115,15 +164,16 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
}
|
||||
|
||||
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
|
||||
// depth compose with the one machine and one case `change`, its manifests, verdicts and seats from the step's revision; the
|
||||
// depth compose with the one machine and one case `change`, its manifests and seats from the step's revision, no verdicts (H-A-112); the
|
||||
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
|
||||
// trunk's; the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
|
||||
// trunk's, or the revision's for a machine whose node-engine is not heard running (A-451); the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
|
||||
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
out := s.r.run(context.Background(), "ask-step", step)
|
||||
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
|
||||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
|
||||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
|
||||
out.Warnings[1] != "g14: the composer names no culprits" {
|
||||
t.Fatalf("the step came to %+v", out)
|
||||
}
|
||||
if len(s.asks) != 2 {
|
||||
@@ -137,14 +187,20 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
if s.asks["g14"].Run.Program.Commit != commitE2 {
|
||||
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
|
||||
}
|
||||
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
// Each composer is built against the pins of the node-engine entry it is built from, a pin with no version left out
|
||||
// (H-A-203, H-A-192).
|
||||
for node, fp := range map[string]string{"ace": "src2:e1", "g14": "src2:e2"} {
|
||||
if got := s.asks[node].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25-"+fp {
|
||||
t.Errorf("%s's composer is built against the pins %v; want its entry's", node, got)
|
||||
}
|
||||
}
|
||||
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("the request of ace %+v", req)
|
||||
}
|
||||
c := req.Cases[0]
|
||||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
|
||||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
|
||||
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
|
||||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" {
|
||||
t.Errorf("the case %+v", c.Manifests)
|
||||
}
|
||||
if strings.Join(s.entries, " ") == "" {
|
||||
t.Error("no node-engine entry is read through entry")
|
||||
@@ -159,6 +215,18 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
|
||||
}
|
||||
|
||||
// No running of the node-engine is heard on g14: its composer is the revision's entry, and no entry is asked for it (A-451).
|
||||
unheard := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
unheard.r.running = func(node, module string) (ne.Running, bool) {
|
||||
return ne.Running{Node: node, Module: module, Fingerprint: "src2:e1"}, node == "ace" && module == "engine"
|
||||
}
|
||||
if out := unheard.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") {
|
||||
t.Fatalf("the step with g14's node-engine not heard came to %+v", out)
|
||||
}
|
||||
if unheard.asks["g14"].Run == nil || unheard.asks["g14"].Run.Program.Commit != commitE1 {
|
||||
t.Errorf("g14's composer with no running heard is %+v, want the revision's entry", unheard.asks["g14"].Run)
|
||||
}
|
||||
|
||||
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
|
||||
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
|
||||
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
@@ -166,6 +234,147 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
|
||||
}
|
||||
|
||||
// A revision naming no definition (A-234's stand-in): the composer is handed the trunk seat list's definitions, read through
|
||||
// seat-definitions (A-467); a list that cannot be read, or names no seat, ends the step could not run, naming it, with no ask put.
|
||||
bare := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
bare.noDefinitions, bare.trunkSeats = true, []meshcatalog.SeatDefinitionsAnswerSeatsItem{{Seat: "s1", Digest: "d1"}}
|
||||
if out := bare.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
len(bare.requests["ace"].Seats) != 1 || string(bare.requests["ace"].Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("a revision with no definitions came to %+v, seats %v", out, bare.requests["ace"].Seats)
|
||||
}
|
||||
for name, fault := range map[string]error{"unreadable": errors.New("no responder"), "empty": nil} {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.noDefinitions, s.listFault = true, fault
|
||||
if out := s.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "seat list is not read") || len(s.asks) != 0 {
|
||||
t.Errorf("the %s seat list came to %+v, %d asks", name, out, len(s.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the composer did not judge is could not run, naming why: a step that judged nothing never passes (A-467).
|
||||
nj := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
nj.unjudged = "g14"
|
||||
if out := nj.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "g14 was not judged: the composer was handed no seat list") {
|
||||
t.Errorf("a machine not judged came to %+v", out)
|
||||
}
|
||||
|
||||
// No machine judged: every machine away, or no machine, never passes; each is named with why (A-467a).
|
||||
for name, machines := range map[string][]string{"every machine away": {"shanks"}, "no machine": {}} { // none placed: the inputs say so already
|
||||
want := "no machine was judged"
|
||||
if len(machines) == 0 {
|
||||
want = "has no machine to compose"
|
||||
}
|
||||
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
none.r.inputs.machines = func(string, string) []string { return machines }
|
||||
outcome := meshwants.StepEndedOutcomeCouldNotRun
|
||||
if len(machines) == 0 { // a mover no machine holds: nothing to run, saying why (H-A-179)
|
||||
outcome = meshwants.StepEndedOutcomeNothingToRun
|
||||
}
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != outcome ||
|
||||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
|
||||
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in
|
||||
// the machine's reason (H-A-143, §532).
|
||||
across := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
across.r.newest = func() map[string]mcc.Half {
|
||||
return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}}
|
||||
}
|
||||
across.r.watched = func(module string) []string {
|
||||
if module == "checker" || module == "lidarr" { // no registration names them; lidarr is on main's revision all the same (H-A-144)
|
||||
return nil
|
||||
}
|
||||
return []string{"main", "master"}
|
||||
}
|
||||
across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker", "lidarr"} }
|
||||
if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
!strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") ||
|
||||
!strings.Contains(out.Said, "g14 is assigned lidarr, which no watched branch lists (H-A-143)") {
|
||||
t.Errorf("the step across the watched branches came to %+v", out)
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 4 ||
|
||||
c.Manifests["lidarr"].Raw != nil {
|
||||
t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests)))
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; c.Assignments != nil {
|
||||
t.Errorf("with no assignments held the case composes the input's, it carries %v", c.Assignments)
|
||||
}
|
||||
|
||||
// A stale assignment is left out of the case's assignments as well, said, and the machine composes; a listed module stays in
|
||||
// (H-A-148, ADR 0578 §2). The one set is the watched branches' newest revisions', carried on the run (H-A-152, H-A-153): a module
|
||||
// the candidate drops while the newest still lists it stays in, the change's own fault, and while no newest revision is followed
|
||||
// nothing is left out.
|
||||
dropsWeb := func() map[string]mcc.Half {
|
||||
h := across.r.newest()
|
||||
h["main"] = mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: "main", Revision: 6},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Manifest: map[string]any{"module": "m"}},
|
||||
"web": {Module: "web", Fp: "w1", Manifest: map[string]any{"module": "web"}}}}
|
||||
return h
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string
|
||||
newest func() map[string]mcc.Half // nil: across's
|
||||
want []string // the case's modules of ace; nil: the input's assignments
|
||||
}{
|
||||
{"a stale assignment left out and said, the machine composes", []string{"m", "f1-circuits", "checker"}, nil, []string{"m", "f1-circuits"}},
|
||||
{"listed modules alone: the input's assignments", []string{"m", "f1-circuits"}, nil, nil},
|
||||
{"a module the candidate drops, the newest lists: kept", []string{"m", "f1-circuits", "web"}, dropsWeb, nil},
|
||||
{"no newest revision followed: nothing left out", []string{"m", "checker"}, func() map[string]mcc.Half { return map[string]mcc.Half{} }, nil},
|
||||
} {
|
||||
left := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
left.r.newest, left.r.watched = across.r.newest, across.r.watched
|
||||
if tc.newest != nil {
|
||||
left.r.newest = tc.newest
|
||||
}
|
||||
left.r.assigned = func(string) []string { return tc.assigned }
|
||||
left.r.assignment = func(node string) (mma.Assignments, bool) {
|
||||
a := mma.Assignments{Node: node}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
return a, true
|
||||
}
|
||||
left.verdict = func(string) (composer.Machine, bool) { return composer.Machine{Composes: true}, true }
|
||||
raised := map[string][]byte{}
|
||||
left.r.findings = assign.Findings{Put: putInto(raised)}
|
||||
out := left.r.run(context.Background(), "ask-step", step)
|
||||
// H-A-149: the step that leaves an assignment out raises assignment-unlisted, scope delivery, id <node>.<module>, a warning.
|
||||
var f mcseat.Finding
|
||||
_ = json.Unmarshal(raised[assign.UnlistedKey("ace", "checker")], &f)
|
||||
if want := tc.want != nil; want != (f.Kind == assign.KindAssignmentUnlisted && f.Scope == "delivery" && f.ID == "ace.checker" &&
|
||||
f.Severity == mcseat.FindingSeverityWarning) || (!want && len(raised) != 0) {
|
||||
t.Errorf("%s: raised %v", tc.name, raised)
|
||||
}
|
||||
said := "ace is assigned checker, which no watched branch lists (H-A-143)"
|
||||
if out.Outcome != meshwants.StepEndedOutcomePassed || (tc.want != nil) != slices.Contains(out.Warnings, said) {
|
||||
t.Errorf("%s: the step came to %+v", tc.name, out)
|
||||
}
|
||||
if run, _, _, _ := left.runs.read(runKeyOf("main.7", "ace")); (tc.want != nil) != slices.Equal(run.Unlisted, []string{"checker"}) ||
|
||||
(tc.want == nil && run.Unlisted != nil) {
|
||||
t.Errorf("%s: the run carries the unlisted %v", tc.name, run.Unlisted)
|
||||
}
|
||||
raw, ok := left.requests["ace"].Cases[0].Assignments["ace"]
|
||||
if tc.want == nil {
|
||||
if ok {
|
||||
t.Errorf("%s: the case carries assignments %s", tc.name, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
a, err := mmac.DecodeAssignments(raw)
|
||||
var got []string
|
||||
for _, m := range a.Modules {
|
||||
got = append(got, m.Module)
|
||||
}
|
||||
if err != nil || !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: the case composes ace with %v (%v), want %v", tc.name, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// At the inputs state's cap the ask is refused, naming the store (§354).
|
||||
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
|
||||
@@ -174,3 +383,425 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
|
||||
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
|
||||
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
faults int // consecutive faults of the `revision` read before it answers
|
||||
fault error // the fault
|
||||
steps int
|
||||
want string // the outcome of the last step
|
||||
revisionCalls int
|
||||
waits int
|
||||
}{
|
||||
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
|
||||
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
|
||||
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
|
||||
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
inner := s.r.call
|
||||
var mu sync.Mutex
|
||||
calls, faults := 0, tc.faults
|
||||
s.r.call = func(key string, body any) (json.RawMessage, error) {
|
||||
if key == mcc.RevisionVerb {
|
||||
mu.Lock()
|
||||
calls++
|
||||
f := faults > 0
|
||||
faults--
|
||||
mu.Unlock()
|
||||
if f {
|
||||
return nil, tc.fault
|
||||
}
|
||||
}
|
||||
return inner(key, body)
|
||||
}
|
||||
clock := time.Unix(1000, 0)
|
||||
var waits []time.Duration
|
||||
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
|
||||
s.r.sleep = func(_ context.Context, d time.Duration) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
waits = append(waits, d)
|
||||
clock = clock.Add(d)
|
||||
return nil
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, tc.steps)
|
||||
for i := range outs {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
|
||||
}
|
||||
wg.Wait()
|
||||
for _, o := range outs {
|
||||
if o.Outcome != tc.want {
|
||||
t.Fatalf("a step came to %+v, want %s", o, tc.want)
|
||||
}
|
||||
}
|
||||
if tc.waits == -1 { // never answers: bounded, growing waits
|
||||
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
|
||||
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
|
||||
}
|
||||
var total time.Duration
|
||||
for _, w := range waits {
|
||||
total += w
|
||||
}
|
||||
if total > readsBound {
|
||||
t.Errorf("read again for %s, bound %s", total, readsBound)
|
||||
}
|
||||
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
|
||||
t.Errorf("the step said %q", outs[0].Said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if calls != tc.revisionCalls || len(waits) != tc.waits {
|
||||
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
|
||||
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
|
||||
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
|
||||
// own runs, and a run that could not run is never read again.
|
||||
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, 3)
|
||||
for i, m := range []string{"m", "m2", "m3"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
|
||||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
|
||||
t.Errorf("step %d names the runs %+v", i, o.Shared)
|
||||
}
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
|
||||
t.Errorf("step %d came to %+v, want both machines read", i, o)
|
||||
}
|
||||
}
|
||||
if s.puts != 2 {
|
||||
t.Fatalf("%d run asks put, want one per machine", s.puts)
|
||||
}
|
||||
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
|
||||
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
|
||||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
|
||||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
|
||||
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
|
||||
}
|
||||
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
|
||||
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
|
||||
}
|
||||
// Another candidate (a run on request) starts its own runs.
|
||||
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
|
||||
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
|
||||
}
|
||||
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
|
||||
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
|
||||
put, refused := f.r.put, 0
|
||||
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
|
||||
if refused == 0 {
|
||||
refused++
|
||||
return errors.New("no stream")
|
||||
}
|
||||
return put(ctx, a)
|
||||
}
|
||||
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
|
||||
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
|
||||
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
|
||||
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
|
||||
// try is could not run for its own step; a run that could not run is could not run.
|
||||
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
|
||||
at := time.Now()
|
||||
runOf := func(verdict string) meshwants.CompositionRun {
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
|
||||
}
|
||||
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
|
||||
for _, tc := range []struct {
|
||||
name, module string
|
||||
run meshwants.CompositionRun
|
||||
failed, couldNo string
|
||||
warnings []string
|
||||
unlisted []string
|
||||
}{
|
||||
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil},
|
||||
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil},
|
||||
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil},
|
||||
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil},
|
||||
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil},
|
||||
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
|
||||
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil},
|
||||
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
|
||||
[]string{"ace holds a: not tried: the search reached its bound"}, nil},
|
||||
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil},
|
||||
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil},
|
||||
// H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes.
|
||||
{"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`),
|
||||
"ace does not compose and blames no module: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"does not compose, blames a module, an unlisted module", "a", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[{"module":"a","problem":"nothing provides \"photos\""}]}`),
|
||||
"ace holds a: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"another module blamed, an unlisted module", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"x"}]}`), "", "",
|
||||
[]string{"ace holds a: x", "ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
{"could not run, an unlisted module", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"},
|
||||
"", "ace: gone; ace is assigned photos, which no watched branch lists (H-A-143)", nil, []string{"photos"}},
|
||||
{"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "",
|
||||
[]string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
run := tc.run
|
||||
run.Unlisted = tc.unlisted // the set the run's start wrote (H-A-153)
|
||||
e := judgeRun(tc.module, run)
|
||||
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
|
||||
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
|
||||
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
|
||||
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
|
||||
// step passed keeps its verdict.
|
||||
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var mu sync.Mutex
|
||||
fixed := false
|
||||
s.verdict = func(node string) (composer.Machine, bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
m.Blamed = []struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{}
|
||||
if node == "g14" && !fixed {
|
||||
m.Composes, m.Reason = false, "two modules own one path"
|
||||
m.Blamed = append(m.Blamed, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"a", "/etc/x"}, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"d", "/etc/x"})
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
ends := make(chan meshwants.StepEnded, 16)
|
||||
msgIDs := map[string]string{} // the message id of each step's latest end
|
||||
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
|
||||
say: func(event string, body any, msgID string) error {
|
||||
if e, ok := body.(meshwants.StepEnded); ok {
|
||||
mu.Lock()
|
||||
msgIDs[e.ID] = msgID
|
||||
mu.Unlock()
|
||||
ends <- e
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
move := steps.move(context.Background())
|
||||
s.runs.moved.Store(&move)
|
||||
step := func(m string, rerun bool) []byte {
|
||||
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
|
||||
return raw
|
||||
}
|
||||
got := map[string]meshwants.StepEnded{}
|
||||
for _, m := range []string{"a", "b", "d"} {
|
||||
pullOne(steps, "ask-"+m, step(m, false))
|
||||
e := <-ends
|
||||
got[e.Module] = e
|
||||
}
|
||||
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the first ends: %+v", got)
|
||||
}
|
||||
former := got["d"].Shared[1].Ask
|
||||
mu.Lock()
|
||||
fixed = true
|
||||
mu.Unlock()
|
||||
pullOne(steps, "ask-a2", step("a", true))
|
||||
moved := map[string]meshwants.StepEnded{}
|
||||
for range 2 {
|
||||
select {
|
||||
case e := <-ends:
|
||||
moved[e.ID] = e
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends after the rerun: %+v", moved)
|
||||
}
|
||||
}
|
||||
a, d := moved["ask-a2"], moved["ask-d"]
|
||||
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
|
||||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
|
||||
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
|
||||
}
|
||||
mu.Lock()
|
||||
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
|
||||
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
|
||||
t.Errorf("the message ids of the ends: %v", msgIDs)
|
||||
}
|
||||
mu.Unlock()
|
||||
if s.puts != 3 {
|
||||
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
|
||||
}
|
||||
select {
|
||||
case e := <-ends:
|
||||
t.Errorf("a reader that passed moved: %+v", e)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
marks := map[string]bool{}
|
||||
for _, c := range run.Readers {
|
||||
marks[c.ID] = c.Rerun
|
||||
}
|
||||
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
|
||||
t.Errorf("the new run's readers %+v", run.Readers)
|
||||
}
|
||||
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
wake := s.runs.wake()
|
||||
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-wake:
|
||||
case <-deadline:
|
||||
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its
|
||||
// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the
|
||||
// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision.
|
||||
func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) {
|
||||
entry := func(module, fp string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}}
|
||||
}
|
||||
half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half {
|
||||
h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: branch}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, m := range movers {
|
||||
h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m})
|
||||
}
|
||||
for _, e := range entries {
|
||||
h.Entries[e.Module] = e
|
||||
}
|
||||
return h
|
||||
}
|
||||
newest := map[string]mcc.Half{
|
||||
"main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")),
|
||||
"master": half("master", 9, nil, entry("f1-circuits", "f1")),
|
||||
"nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")),
|
||||
}
|
||||
registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}}
|
||||
registrations["engine"], registrations["a"] = []string{"main"}, []string{"main"}
|
||||
watched := func(module string) []string { return registrations[module] } // a module no registration names has none (H-A-144)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
step meshwants.StepRun
|
||||
candidate mcc.Half
|
||||
newest map[string]mcc.Half
|
||||
want map[string]string // module -> fingerprint
|
||||
}{
|
||||
{"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}},
|
||||
{"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2},
|
||||
half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}},
|
||||
{"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1},
|
||||
half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]},
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}},
|
||||
{"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}},
|
||||
{"a module no registration names is not composed", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2"), entry("checker", "c1")), nil, map[string]string{"a": "a2"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := acrossBranches(tc.step, tc.candidate, tc.newest, watched)
|
||||
fps := map[string]string{}
|
||||
for name, e := range got.Entries {
|
||||
fps[name] = e.Fp
|
||||
}
|
||||
if !maps.Equal(fps, tc.want) {
|
||||
t.Errorf("composed from %v, want %v", fps, tc.want)
|
||||
}
|
||||
if got.Revision.Revision != tc.candidate.Revision.Revision || len(got.Revision.Definitions) != 1 || got.Revision.Definitions[0].Digest != tc.step.Branch {
|
||||
t.Errorf("the half keeps revision %d with definitions %v, want the candidate's %d with its own", got.Revision.Revision, got.Revision.Definitions,
|
||||
tc.candidate.Revision.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-149: the pass over the assignments clears assignment-unlisted when the assignment is gone or one of the module's
|
||||
// watched branches lists it again, and keeps it while it stands; another finding is left alone.
|
||||
func TestThePassClearsAnUnlistedAssignmentsFinding(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string // novox's modules
|
||||
listed bool
|
||||
kept bool
|
||||
}{
|
||||
{"still assigned and listed nowhere: kept", []string{"checker"}, false, true},
|
||||
{"the assignment gone: cleared", []string{"web"}, false, false},
|
||||
{"listed again: cleared", []string{"checker"}, true, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := newWorld()
|
||||
a := mma.Assignments{V: keys.Version, Node: "novox"}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
putAssignments(w, a)
|
||||
findings := map[string][]byte{}
|
||||
f, _ := assign.UnlistedFinding("novox", "checker", time.Now())
|
||||
_ = putInto(findings)(f)
|
||||
other, _ := assign.UnplacedFinding("g14", time.Now())
|
||||
_ = putInto(findings)(other)
|
||||
p := &passer{w: w, logf: t.Logf, listed: func(string) bool { return tc.listed }, findings: assign.Findings{Put: putInto(findings),
|
||||
Delete: func(k string) error { delete(findings, k); return nil },
|
||||
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(findings)), nil }}}
|
||||
p.clearUnlisted()
|
||||
if _, ok := findings[assign.UnlistedKey("novox", "checker")]; ok != tc.kept {
|
||||
t.Errorf("the finding kept %v, want %v", ok, tc.kept)
|
||||
}
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok {
|
||||
t.Error("another finding is cleared")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,916 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
|
||||
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
|
||||
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
|
||||
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
|
||||
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
|
||||
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
|
||||
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
|
||||
|
||||
const (
|
||||
// runsSwept is how many of the oldest ended keys each new run judges.
|
||||
runsSwept = 20
|
||||
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
|
||||
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
|
||||
runBoundPrefix = "bound."
|
||||
runUntakenPrefix = "untaken."
|
||||
)
|
||||
|
||||
// runKeyOf is the key of a candidate's composition run of a machine (§719).
|
||||
func runKeyOf(candidate, machine string) string {
|
||||
return keysPart(candidate) + "." + keys.EscapeName(machine)
|
||||
}
|
||||
|
||||
// keysPart is a name as a key's part.
|
||||
func keysPart(name string) string { return keys.EscapeName(name) }
|
||||
|
||||
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
|
||||
func isRunKey(key string) bool {
|
||||
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
|
||||
}
|
||||
|
||||
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
|
||||
type runSettings struct {
|
||||
// capBytes is composition-runs-cap-mib in bytes.
|
||||
capBytes int64
|
||||
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
|
||||
kept, stepBound, untaken time.Duration
|
||||
// blameTries is blame-tries, carried in each run's request (§721).
|
||||
blameTries int
|
||||
}
|
||||
|
||||
// defaultRunSettings are the settings' defaults, as the manifest gives them.
|
||||
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
|
||||
|
||||
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
|
||||
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
|
||||
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
|
||||
func runSettingsOf(getenv func(string) string) (runSettings, error) {
|
||||
s := defaultRunSettings
|
||||
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if mib > 0 {
|
||||
s.capBytes = int64(mib) << 20
|
||||
}
|
||||
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if tries > 0 {
|
||||
s.blameTries = tries
|
||||
}
|
||||
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
|
||||
d, err := durationFile(getenv(env))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if d > 0 {
|
||||
*to = d
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
|
||||
func durationFile(path string) (time.Duration, error) {
|
||||
if path == "" {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return 0, nil
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// errRunChanged is a compare-and-set of a run's key that met another write.
|
||||
var errRunChanged = errors.New("the composition run's key changed")
|
||||
|
||||
// runsKV is the state composition-runs: the bus bucket, or a test's.
|
||||
type runsKV interface {
|
||||
// get is a key's value and revision; nil for a key not there.
|
||||
get(key string) ([]byte, uint64, error)
|
||||
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
|
||||
create(key string, value []byte) (uint64, error)
|
||||
update(key string, value []byte, revision uint64) (uint64, error)
|
||||
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
|
||||
// of the state is made from a value read before its latest change.
|
||||
delAt(key string, revision uint64) error
|
||||
keys() ([]string, error)
|
||||
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
|
||||
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
|
||||
// answers the revision written.
|
||||
arm(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// used is how many bytes the state holds.
|
||||
used() (int64, error)
|
||||
}
|
||||
|
||||
// busRuns is runsKV over the bus bucket.
|
||||
type busRuns struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b busRuns) get(key string) ([]byte, uint64, error) {
|
||||
e, err := b.kv.Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (b busRuns) create(key string, value []byte) (uint64, error) {
|
||||
rev, err := b.kv.Create(b.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := b.kv.Update(b.ctx, key, value, revision)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) delAt(key string, revision uint64) error {
|
||||
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return errRunChanged
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busRuns) keys() ([]string, error) {
|
||||
l, err := b.kv.ListKeys(b.ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b.js == nil {
|
||||
return 0, errors.New("no bus to arm a bound on")
|
||||
}
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject := "$KV." + b.kv.Bucket() + "." + key
|
||||
for {
|
||||
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var last uint64
|
||||
m, err := st.GetLastMsgForSubject(b.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b busRuns) used() (int64, error) {
|
||||
st, err := b.kv.Status(b.ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(st.Bytes()), nil
|
||||
}
|
||||
|
||||
// compositionRuns keeps the composition runs in the state composition-runs.
|
||||
type compositionRuns struct {
|
||||
kv runsKV
|
||||
set runSettings
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
|
||||
changed chan struct{}
|
||||
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
|
||||
// being written at that moment, so every lookup takes these beside it (keys).
|
||||
seen map[string]bool
|
||||
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
|
||||
// nil says or drops nothing.
|
||||
say publisher
|
||||
drop func(ask, why string) error
|
||||
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
|
||||
// Set once the steps are served, after the hub starts.
|
||||
moved atomic.Pointer[func(run meshwants.CompositionRun)]
|
||||
}
|
||||
|
||||
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
|
||||
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
|
||||
func (r *compositionRuns) heard() {
|
||||
r.mu.Lock()
|
||||
close(r.changed)
|
||||
r.changed = make(chan struct{})
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// saw keeps a key the state's watch handed: standing, or deleted or expired.
|
||||
func (r *compositionRuns) saw(key string, standing bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.seen == nil {
|
||||
r.seen = map[string]bool{}
|
||||
}
|
||||
if standing {
|
||||
r.seen[key] = true
|
||||
} else {
|
||||
delete(r.seen, key)
|
||||
}
|
||||
}
|
||||
|
||||
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
|
||||
// is skipped by its reader.
|
||||
func (r *compositionRuns) keys() ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
for key := range r.seen {
|
||||
all = append(all, key)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
slices.Sort(all)
|
||||
return slices.Compact(all), nil
|
||||
}
|
||||
|
||||
// wake is closed at the next change heard.
|
||||
func (r *compositionRuns) wake() <-chan struct{} {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.changed
|
||||
}
|
||||
|
||||
// read is a run's value and revision, or false.
|
||||
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(key)
|
||||
if err != nil || raw == nil {
|
||||
return meshwants.CompositionRun{}, 0, false, err
|
||||
}
|
||||
var run meshwants.CompositionRun
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
|
||||
}
|
||||
return run, rev, true, nil
|
||||
}
|
||||
|
||||
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
|
||||
// It answers the value written or left, and false for a key not there.
|
||||
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok {
|
||||
return run, ok, err
|
||||
}
|
||||
if !f(&run) {
|
||||
return run, true, nil
|
||||
}
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
r.heard()
|
||||
return run, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fill is the state's fill against its cap.
|
||||
func (r *compositionRuns) fill() (meshwants.Fill, error) {
|
||||
at := r.now().UTC()
|
||||
used, err := r.kv.used()
|
||||
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
|
||||
}
|
||||
|
||||
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
|
||||
// value as it stood.
|
||||
type joined struct {
|
||||
key string
|
||||
started bool
|
||||
run meshwants.CompositionRun
|
||||
}
|
||||
|
||||
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
|
||||
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
|
||||
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
|
||||
// cap no run is started, an error naming the fill.
|
||||
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
return r.joinAt(runKeyOf(candidate, machine), reader, candidate, machine, ask)
|
||||
}
|
||||
|
||||
// joinAt is join of the run under key: a machine's composition run, or a revision's validate run with no machine (H-A-108).
|
||||
func (r *compositionRuns) joinAt(key string, reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
|
||||
if hasReader(run.Readers, reader.ID) {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
run.Readers = append(run.Readers, reader)
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) {
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
|
||||
Readers: []meshwants.CompositionReader{reader}}
|
||||
raw, err := json.Marshal(fresh)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok {
|
||||
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
|
||||
} else {
|
||||
_, err = r.kv.create(key, raw)
|
||||
}
|
||||
if errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
|
||||
func (r *compositionRuns) room() error {
|
||||
if err := r.sweep(); err != nil {
|
||||
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
|
||||
}
|
||||
f, err := r.fill()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
if f.Bytes >= f.Cap {
|
||||
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
|
||||
f.Cap>>20, f.Bytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
|
||||
func (r *compositionRuns) sweep() error {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type endedAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var ended []endedAt
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended == nil {
|
||||
continue
|
||||
}
|
||||
ended = append(ended, endedAt{key, *run.Ended})
|
||||
}
|
||||
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
|
||||
if len(ended) > runsSwept {
|
||||
ended = ended[:runsSwept]
|
||||
}
|
||||
var errs []error
|
||||
for _, e := range ended {
|
||||
if r.now().Sub(e.at) >= r.set.kept {
|
||||
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
|
||||
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
|
||||
}))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
|
||||
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
|
||||
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok || !gone(run) {
|
||||
return err
|
||||
}
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
|
||||
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
|
||||
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
|
||||
var errs []error
|
||||
for _, prefix := range prefixes {
|
||||
for {
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if ok && run.Ask != ask {
|
||||
break // a newer run holds the key: its bounds are its own
|
||||
}
|
||||
held, rev, standing, err := r.boundOf(prefix + key)
|
||||
if err != nil || !standing {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if held != ask {
|
||||
break // the bound of another ask (A-470n): not this run's to delete
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
|
||||
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
|
||||
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
|
||||
keys, err := r.keysOfStep(step, id)
|
||||
if err != nil {
|
||||
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
|
||||
return
|
||||
}
|
||||
at = at.UTC()
|
||||
for _, key := range keys {
|
||||
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
c.Readers[i].Ended = &at
|
||||
return true
|
||||
}); err != nil {
|
||||
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keysOfStep are the keys of the runs the step asked under id reads: its revision's validate run for a validate step, its
|
||||
// candidate's runs naming it for a composition step.
|
||||
func (r *compositionRuns) keysOfStep(step meshwants.StepRun, id string) ([]string, error) {
|
||||
if step.Step == meshwants.StepRunStepValidate {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{key}, nil
|
||||
}
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.keysOf(candidate, id)
|
||||
}
|
||||
|
||||
// hasReader is whether readers holds the step asked under id.
|
||||
func hasReader(readers []meshwants.CompositionReader, id string) bool {
|
||||
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
}
|
||||
|
||||
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
|
||||
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
|
||||
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
|
||||
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
|
||||
}
|
||||
|
||||
// stepRunOf is a reader's StepRun again, for its step-ended.
|
||||
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
|
||||
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
|
||||
}
|
||||
|
||||
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
|
||||
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
|
||||
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
if ready != nil {
|
||||
ready()
|
||||
ready = nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
|
||||
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
|
||||
expired(e.Key())
|
||||
}
|
||||
r.heard()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
|
||||
// false when the key holds another run or the run has ended already.
|
||||
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
ended := false
|
||||
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
at := r.now().UTC()
|
||||
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
|
||||
ended = true
|
||||
return true
|
||||
})
|
||||
return ended, err
|
||||
}
|
||||
|
||||
// keyOfAsk is the key of the run whose ask is ask, or none.
|
||||
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err == nil && ok && run.Ask == ask {
|
||||
return key, run, nil
|
||||
}
|
||||
}
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
|
||||
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
|
||||
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
|
||||
// names is another asker's, or of a run gone, and changes nothing.
|
||||
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
|
||||
if e.Asker != runAsker || e.Ask == "" {
|
||||
return nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(e.Ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
|
||||
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
|
||||
return nil
|
||||
}
|
||||
node := run.Machine
|
||||
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
|
||||
var verdict map[string]any
|
||||
if e.Outcome != buildstep.OutcomePassed {
|
||||
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
|
||||
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
|
||||
said = node + ": " + err.Error()
|
||||
} else if isValidateRun(run) {
|
||||
if verdict, said = validateRunVerdict(answer); said == "" {
|
||||
outcome = meshwants.CompositionRunOutcomePassed
|
||||
}
|
||||
} else if verdict, said = runVerdict(node, answer); said == "" {
|
||||
outcome = meshwants.CompositionRunOutcomePassed
|
||||
}
|
||||
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
|
||||
return err
|
||||
}
|
||||
|
||||
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
|
||||
// stands then; a key gone could not run.
|
||||
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
|
||||
out := map[string]meshwants.CompositionRun{}
|
||||
for {
|
||||
wake := r.wake()
|
||||
open := false
|
||||
for _, key := range keys {
|
||||
run, _, ok, err := r.read(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
open = true
|
||||
continue
|
||||
case !ok:
|
||||
at := r.now().UTC()
|
||||
candidate, machine, _ := strings.Cut(key, ".")
|
||||
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
|
||||
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
|
||||
case run.Ended == nil:
|
||||
open = true
|
||||
}
|
||||
out[key] = run
|
||||
}
|
||||
if !open {
|
||||
return out
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return out
|
||||
case <-wake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unescape is a key's part as a name again; a part that does not read stays as it is.
|
||||
func unescape(part string) string {
|
||||
if name, err := keys.UnescapeName(part); err == nil {
|
||||
return name
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
|
||||
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
|
||||
out := []meshwants.Shared{}
|
||||
for _, key := range keys {
|
||||
run, ok := runs[key]
|
||||
if !ok {
|
||||
if run, _, ok, _ = r.read(key); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
|
||||
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
|
||||
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
|
||||
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
var errs []error
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
var left []string
|
||||
var leaving []meshwants.CompositionReader
|
||||
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
left, leaving = nil, nil
|
||||
keep := c.Readers[:0:0]
|
||||
for _, reader := range c.Readers {
|
||||
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
|
||||
left = append(left, reader.ID)
|
||||
leaving = append(leaving, reader)
|
||||
continue
|
||||
}
|
||||
keep = append(keep, reader)
|
||||
}
|
||||
c.Readers = keep
|
||||
return len(left) > 0
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if !ok || len(left) == 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, left...)
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
|
||||
r.sayWaits(run, leaving, "superseded")
|
||||
}
|
||||
if stop != nil {
|
||||
stop(left) // the steps that left say no end, the run's included
|
||||
}
|
||||
if run.Ended != nil || len(run.Readers) > 0 {
|
||||
continue
|
||||
}
|
||||
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, why); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return ids, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
|
||||
func passedOn(module string, run meshwants.CompositionRun) bool {
|
||||
e := judgeVerdict(module, run) // the unlisted modules change only the words, never whether it passed
|
||||
return e.failed == "" && e.couldNot == ""
|
||||
}
|
||||
|
||||
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
|
||||
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
|
||||
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
|
||||
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
|
||||
})
|
||||
}
|
||||
|
||||
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
|
||||
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
|
||||
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
|
||||
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
|
||||
// not (A-470d).
|
||||
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
key := runKeyOf(candidate, machine)
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if !ok {
|
||||
return r.join(reader, candidate, machine, ask)
|
||||
}
|
||||
at := formerOf(run.Readers, reader.Pipeline)
|
||||
ended := run.Ended != nil
|
||||
keep := ended && at >= 0 && passedOn(reader.Module, run)
|
||||
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
|
||||
if keep || read {
|
||||
if at >= 0 && run.Readers[at].ID == reader.ID {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
var former []meshwants.CompositionReader
|
||||
if at >= 0 {
|
||||
former = []meshwants.CompositionReader{run.Readers[at]}
|
||||
run.Readers[at] = reader
|
||||
} else {
|
||||
run.Readers = append(run.Readers, reader)
|
||||
}
|
||||
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
|
||||
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
|
||||
for i, c := range run.Readers {
|
||||
if i == at {
|
||||
continue
|
||||
}
|
||||
c.Rerun, c.By = !passedOn(c.Module, run), ""
|
||||
if c.Rerun {
|
||||
c.By = reader.By
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, c)
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, reader)
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
|
||||
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// write writes a run's value at its revision.
|
||||
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
+404
@@ -0,0 +1,404 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
|
||||
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
|
||||
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
|
||||
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
|
||||
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
|
||||
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
|
||||
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
|
||||
// pass and moves b; a superseded step leaves
|
||||
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
|
||||
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
|
||||
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
|
||||
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const commit = "1111111111111111111111111111111111111111"
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
awaitKept(t, b, nodeengine.StreamRunning, 2)
|
||||
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
|
||||
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
|
||||
}
|
||||
b.cat.rev[trunkBranch] = 3
|
||||
b.cat.mu.Unlock()
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.place(t, node, m)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
inputsAsked := map[string]int{} // by candidate and node
|
||||
var drops []string
|
||||
respond := func(subject string, f func(m *nats.Msg) any) {
|
||||
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
|
||||
node := strings.Split(m.Subject, ".")[5]
|
||||
var in nodeengine.ComposeCandidateInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
inputsAsked[in.Candidate+" "+node]++
|
||||
mu.Unlock()
|
||||
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
|
||||
})
|
||||
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
|
||||
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
|
||||
var in nodebuildgo.DropInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
drops = append(drops, in.ID)
|
||||
mu.Unlock()
|
||||
return map[string]any{}
|
||||
})
|
||||
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
|
||||
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
|
||||
// repeated message id is never heard (A-470q).
|
||||
ended := make(chan meshwants.StepEnded, 32)
|
||||
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
|
||||
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard, err := runner.Consume(func(m jetstream.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data(), &e)
|
||||
_ = m.Ack()
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(heard.Stop)
|
||||
subscribed(t, nc)
|
||||
|
||||
from := b.journal.mark()
|
||||
_, stop := b.startStoppable(t)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
|
||||
select {
|
||||
case <-b.seatListRead:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
|
||||
AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
step := func(id, module, run string, rerun bool) {
|
||||
t.Helper()
|
||||
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
|
||||
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
|
||||
if run != "" {
|
||||
s.Pipeline = "run-" + run + "." + module
|
||||
}
|
||||
if rerun {
|
||||
s.By = "op@n1"
|
||||
}
|
||||
raw, _ := json.Marshal(s)
|
||||
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Data = raw
|
||||
if err := nc.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
type asked struct {
|
||||
ask buildstep.StepAsk
|
||||
machine string
|
||||
}
|
||||
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
|
||||
asks := func(n int) []asked {
|
||||
t.Helper()
|
||||
var out []asked
|
||||
for range n {
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil {
|
||||
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
|
||||
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
||||
}
|
||||
out = append(out, asked{a, req.Machines[0]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
noMoreAsks := func(what string) {
|
||||
t.Helper()
|
||||
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
|
||||
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
|
||||
}
|
||||
}
|
||||
say := func(event string, node string, body any) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(body)
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer := func(a asked, machine string) {
|
||||
t.Helper()
|
||||
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
|
||||
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
|
||||
}
|
||||
ends := func(n int, what string) map[string]meshwants.StepEnded {
|
||||
t.Helper()
|
||||
out := map[string]meshwants.StepEnded{}
|
||||
for len(out) < n {
|
||||
select {
|
||||
case e := <-ended:
|
||||
out[e.ID] = e
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
runKeys := func(candidate string) []string {
|
||||
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
|
||||
var out []string
|
||||
for _, k := range all {
|
||||
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// readers waits, through a watch of the key, until the run of key has n readers.
|
||||
readers := func(key string, n int) {
|
||||
t.Helper()
|
||||
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
var run meshwants.CompositionRun
|
||||
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the run %s has not %d readers", key, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
|
||||
composes := `{"composes":true,"blamed":[]}`
|
||||
|
||||
// One candidate, three steps, two machines: two runs.
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
step("step-"+m, m, "", false)
|
||||
}
|
||||
first := asks(2)
|
||||
noMoreAsks("three steps of one candidate")
|
||||
for _, a := range first {
|
||||
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
|
||||
}
|
||||
got := ends(3, "the three steps")
|
||||
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
|
||||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
|
||||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
|
||||
t.Fatalf("the steps of one candidate: %+v", got)
|
||||
}
|
||||
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
|
||||
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
|
||||
}
|
||||
|
||||
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
|
||||
step("step-r1", "a", "r1", false)
|
||||
for _, a := range asks(2) {
|
||||
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
|
||||
}
|
||||
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
|
||||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
|
||||
t.Fatalf("a step of another candidate: %+v", e)
|
||||
}
|
||||
|
||||
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
|
||||
from = b.journal.mark()
|
||||
step("step-r2", "a", "r2", false)
|
||||
bound := asks(2)
|
||||
var n1 asked
|
||||
for _, a := range bound {
|
||||
if a.machine == "n1" {
|
||||
n1 = a
|
||||
}
|
||||
}
|
||||
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
|
||||
for i := range 3 { // past the 2 s bound, each beat within it
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a run saying its progress ended: %+v", e)
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
|
||||
runs := b.buckets[buckets.CompositionRuns()]
|
||||
all, _ := runs.Keys(b.ctx)
|
||||
held := ""
|
||||
for _, k := range all {
|
||||
if strings.HasPrefix(k, runBoundPrefix) {
|
||||
if e, err := runs.Get(b.ctx, k); err == nil {
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
held += v.Ask
|
||||
}
|
||||
}
|
||||
}
|
||||
if held != n1.ask.ID {
|
||||
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
|
||||
}
|
||||
}
|
||||
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
|
||||
}
|
||||
e := ends(1, "the bounds")["step-r2"]
|
||||
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
|
||||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
|
||||
t.Fatalf("past the bounds: %+v", e)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := len(drops)
|
||||
mu.Unlock()
|
||||
if dropped != 2 {
|
||||
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
|
||||
}
|
||||
answer(n1, composes) // heard after the bound: ignored and journaled
|
||||
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
|
||||
|
||||
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
|
||||
step("step-a2", "a", "", true)
|
||||
again := asks(1)
|
||||
noMoreAsks("a rerun of a")
|
||||
if again[0].machine != "n2" {
|
||||
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
|
||||
}
|
||||
answer(again[0], composes)
|
||||
moved := ends(2, "the rerun")
|
||||
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
|
||||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
|
||||
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
|
||||
}
|
||||
|
||||
// A superseded step leaves the runs; its other reader goes on.
|
||||
step("step-r3a", "a", "r3", false)
|
||||
step("step-r3b", "b", "r3", false)
|
||||
sup := asks(2)
|
||||
readers(runKeyOf("run-r3", "n1"), 2)
|
||||
readers(runKeyOf("run-r3", "n2"), 2)
|
||||
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Branch: trunkBranch, Source: 1, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Movers: []mpr.MoverEnded{{Module: "a", Steps: []mpr.StepResult{}}}})
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range sup {
|
||||
answer(a, composes)
|
||||
}
|
||||
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the other reader: %+v", got)
|
||||
}
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a superseded step said its end: %+v", e)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Restarted during a run: the new assigner takes it up by its ask.
|
||||
step("step-r4", "c", "r4", false)
|
||||
restart := asks(2)
|
||||
stop()
|
||||
for _, a := range restart {
|
||||
answer(a, composes)
|
||||
}
|
||||
from = b.journal.mark()
|
||||
b.startStoppable(t)
|
||||
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
|
||||
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
|
||||
t.Fatalf("the step taken up: %+v", e)
|
||||
}
|
||||
|
||||
// compositions-drop: the readers ended naming the drop, the key deleted.
|
||||
step("step-r5", "a", "r5", false)
|
||||
asks(2)
|
||||
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
|
||||
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
|
||||
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
|
||||
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
|
||||
}
|
||||
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
|
||||
t.Fatalf("a reader of the dropped run: %+v", e)
|
||||
}
|
||||
if keys := runKeys("run-r5"); len(keys) != 1 {
|
||||
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
|
||||
}
|
||||
|
||||
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
|
||||
for i := range 2 {
|
||||
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
step("step-r6", "a", "r6", false)
|
||||
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
|
||||
t.Fatalf("at the cap: %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
|
||||
type runsMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
ttl map[string]time.Duration
|
||||
bytes int64
|
||||
n uint64
|
||||
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
|
||||
beforeDel func()
|
||||
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
|
||||
beforeArm func()
|
||||
}
|
||||
|
||||
func (k *runsMem) another() {
|
||||
k.mu.Lock()
|
||||
f := k.beforeDel
|
||||
k.beforeDel = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
func newRunsMem() *runsMem {
|
||||
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
|
||||
}
|
||||
|
||||
func (k *runsMem) get(key string) ([]byte, uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.vals[key], k.revs[key], nil
|
||||
}
|
||||
|
||||
func (k *runsMem) create(key string, v []byte) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; ok {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.revs[key] != rev {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) delAt(key string, rev uint64) error {
|
||||
k.another()
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; !ok {
|
||||
return nil
|
||||
}
|
||||
if k.revs[key] != rev {
|
||||
return errRunChanged
|
||||
}
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *runsMem) keys() ([]string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
var out []string
|
||||
for key := range k.vals {
|
||||
out = append(out, key)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
f := k.beforeArm
|
||||
k.beforeArm = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.n++
|
||||
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
|
||||
func (k *runsMem) expire(key string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
}
|
||||
|
||||
func (k *runsMem) used() (int64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.bytes > 0 {
|
||||
return k.bytes, nil
|
||||
}
|
||||
var n int64
|
||||
for _, v := range k.vals {
|
||||
n += int64(len(v))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
|
||||
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
|
||||
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
|
||||
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
step := func(id, module string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
|
||||
}
|
||||
t.Run("key", func(t *testing.T) {
|
||||
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
|
||||
t.Errorf("the key is %q, want each part escaped as a key's name", k)
|
||||
}
|
||||
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
|
||||
t.Error("a bound key is taken for a run's or a run's for a bound key")
|
||||
}
|
||||
})
|
||||
t.Run("one run for two steps at once", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var wg sync.WaitGroup
|
||||
got := make([]joined, 2)
|
||||
for i, m := range []string{"a", "b"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
got[i] = j
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
runKeys := func() []string {
|
||||
all, _ := kv.keys()
|
||||
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
|
||||
}
|
||||
if got[0].started == got[1].started {
|
||||
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
|
||||
}
|
||||
keys := runKeys()
|
||||
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
|
||||
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
|
||||
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
|
||||
}
|
||||
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
|
||||
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("could not run is never read", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
|
||||
return true
|
||||
})
|
||||
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
|
||||
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
|
||||
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
|
||||
}
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
|
||||
return true
|
||||
})
|
||||
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
|
||||
t.Errorf("a passed run is not read: %+v", j)
|
||||
}
|
||||
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
|
||||
t.Errorf("a step of another candidate reads the first's run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
kv.bytes = 16 << 20
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
|
||||
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
|
||||
t.Errorf("at the cap: %v, want the fill named", err)
|
||||
}
|
||||
})
|
||||
t.Run("kept", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
at := now
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
|
||||
for _, c := range []string{"main.1", "main.2"} {
|
||||
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := now.Add(-25 * time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
|
||||
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
|
||||
recent := now.Add(-time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
|
||||
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, _ := kv.keys()
|
||||
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
|
||||
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
|
||||
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
|
||||
}
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
|
||||
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
|
||||
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
|
||||
}
|
||||
})
|
||||
t.Run("a listing that misses a key being written", func(t *testing.T) {
|
||||
kv := &missingRuns{runsMem: newRunsMem()}
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.hide = runKeyOf("main.3", "n1")
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
|
||||
t.Fatalf("the stand's listing does not hide the key")
|
||||
}
|
||||
r.saw(kv.hide, true) // as the state's watch hands it
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
|
||||
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
|
||||
}
|
||||
r.saw(kv.hide, false)
|
||||
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
|
||||
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
|
||||
}
|
||||
})
|
||||
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
|
||||
// replacing the key, and its untaken bound, stand.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ended time.Time
|
||||
act func(r *compositionRuns, key string) error
|
||||
}{
|
||||
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
|
||||
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
|
||||
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
if !c.ended.IsZero() {
|
||||
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
|
||||
}
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
|
||||
for {
|
||||
raw, rev, _ := kv.get(key)
|
||||
var run meshwants.CompositionRun
|
||||
_ = json.Unmarshal(raw, &run)
|
||||
if run.Ended == nil {
|
||||
at := now
|
||||
run.Ended, run.Outcome = &at, "failed"
|
||||
}
|
||||
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
|
||||
if _, err := kv.update(key, fresh, rev); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
if err := c.act(r, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
|
||||
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
|
||||
}
|
||||
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
|
||||
t.Error("the untaken bound the rerun's run armed is deleted")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("settings", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
|
||||
env := map[string]string{}
|
||||
for name, content := range files {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = p
|
||||
}
|
||||
s, err := runSettingsOf(func(k string) string { return env[k] })
|
||||
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
|
||||
t.Errorf("settings %+v %v", s, err)
|
||||
}
|
||||
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
|
||||
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
|
||||
t.Errorf("a bound of no unit: %v", err)
|
||||
}
|
||||
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
|
||||
t.Errorf("no files: %+v %v, want the defaults", s, err)
|
||||
}
|
||||
})
|
||||
raw, _ := json.Marshal(step("x", "a"))
|
||||
if !strings.Contains(string(raw), `"id":"x"`) {
|
||||
t.Errorf("a reader names no ask id: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
|
||||
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
|
||||
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
|
||||
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
|
||||
// are taken up at the start.
|
||||
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
reader := func(id string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
|
||||
}
|
||||
type said struct{ event, how, module string }
|
||||
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
|
||||
kv := newRunsMem()
|
||||
var saids []said
|
||||
var drops, journal []string
|
||||
var mu sync.Mutex
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
journal = append(journal, fmt.Sprintf(f, a...))
|
||||
mu.Unlock()
|
||||
})
|
||||
r.say = func(event string, body any, _ string) error {
|
||||
w := body.(meshwants.Wait)
|
||||
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
|
||||
t.Errorf("the wait said %+v", w)
|
||||
}
|
||||
saids = append(saids, said{event, w.How, w.Module})
|
||||
return nil
|
||||
}
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
return kv, r, &saids, &drops, &journal
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
t.Run("taken, progress, bound", func(t *testing.T) {
|
||||
kv, r, saids, drops, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
|
||||
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
|
||||
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, untaken, _ := kv.get(runUntakenPrefix + key)
|
||||
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
|
||||
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
before := kv.revs[runBoundPrefix+key]
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
|
||||
t.Fatalf("a step-progress does not arm the bound again: %v", err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
|
||||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
|
||||
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
|
||||
}
|
||||
if got := judgeRun("a", run); got.couldNot != run.Said {
|
||||
t.Errorf("a reader of the run past its bound: %+v", got)
|
||||
}
|
||||
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
|
||||
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
|
||||
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
|
||||
}) {
|
||||
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
|
||||
}
|
||||
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
|
||||
t.Errorf("the bound keys stay past the run's end: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("untaken", func(t *testing.T) {
|
||||
kv, r, saids, drops, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
|
||||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
|
||||
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
|
||||
}
|
||||
r.expired(runUntakenPrefix + key) // a run ended takes nothing
|
||||
if len(*drops) != 1 {
|
||||
t.Errorf("an ended run is dropped again: %v", *drops)
|
||||
}
|
||||
})
|
||||
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
|
||||
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
|
||||
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
|
||||
return func() {
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
|
||||
kv.expire(runBoundPrefix + key)
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
}
|
||||
taken := func(t *testing.T, r *compositionRuns) {
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, kv *runsMem, r *compositionRuns)
|
||||
ended bool
|
||||
bound string // the ask the bound key holds, "" for none
|
||||
untaken string
|
||||
journal string
|
||||
}{
|
||||
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
taken(t, r)
|
||||
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
|
||||
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
rerun(kv, r)()
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
|
||||
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
|
||||
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
|
||||
}
|
||||
}, false, "", "run-1", ""},
|
||||
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
}, false, "run-1", "", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv, r, _, _, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, kv, r)
|
||||
run, _, _, _ := r.read(key)
|
||||
if (run.Ended != nil) != c.ended {
|
||||
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
|
||||
}
|
||||
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
|
||||
if held, _, _, _ := r.boundOf(prefix + key); held != want {
|
||||
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
|
||||
}
|
||||
}
|
||||
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
|
||||
t.Errorf("the journal %v does not say %q", *journal, c.journal)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
|
||||
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
|
||||
// superseded.
|
||||
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
|
||||
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, r *compositionRuns)
|
||||
want []said
|
||||
}{
|
||||
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("b")
|
||||
again.Rerun = true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b")}},
|
||||
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("a2")
|
||||
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), gone("a"), held("a2")}},
|
||||
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.leave("main.3.a", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b"), gone("a")}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, r, saids, _, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, r)
|
||||
if !slices.Equal(*saids, c.want) {
|
||||
t.Errorf("said %v, want %v", *saids, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("taken up at the start", func(t *testing.T) {
|
||||
_, r, _, _, _ := setup(t)
|
||||
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
|
||||
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ended := now
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
|
||||
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"c", "d"} {
|
||||
c := reader(id)
|
||||
c.Ended, c.Rerun = &ended, id == "d"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
|
||||
t.Fatalf("the readers taken up: %+v %v", readers, err)
|
||||
}
|
||||
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
|
||||
t.Errorf("the runs of reader a: %v", keys)
|
||||
}
|
||||
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
|
||||
// `<id>.<the open run's ask>`.
|
||||
ids := make(chan [2]string, 3)
|
||||
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
|
||||
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
},
|
||||
say: func(_ string, body any, id string) error {
|
||||
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
|
||||
return nil
|
||||
}}
|
||||
steps.takeUp(context.Background(), r)
|
||||
got := map[string]string{}
|
||||
for range 3 {
|
||||
select {
|
||||
case e := <-ids:
|
||||
got[e[0]] = e[1]
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends of the take-up: %v", got)
|
||||
}
|
||||
}
|
||||
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
|
||||
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
|
||||
type missingRuns struct {
|
||||
*runsMem
|
||||
hide string
|
||||
}
|
||||
|
||||
func (k *missingRuns) keys() ([]string, error) {
|
||||
all, err := k.runsMem.keys()
|
||||
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
|
||||
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
|
||||
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
|
||||
|
||||
// The tools that call the two verbs.
|
||||
const (
|
||||
compositionsTool = "assigner_compositions"
|
||||
compositionsDropTool = "assigner_compositions_drop"
|
||||
)
|
||||
|
||||
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
|
||||
// machine named only those, and the state's fill against its cap.
|
||||
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
|
||||
continue
|
||||
}
|
||||
run.Key = key
|
||||
out.Runs = append(out.Runs, run)
|
||||
}
|
||||
out.Fill, err = r.fill()
|
||||
return out, err
|
||||
}
|
||||
|
||||
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
|
||||
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
|
||||
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
|
||||
// the readers it ended.
|
||||
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
|
||||
why := strings.TrimSpace(in.Why)
|
||||
if in.Candidate == "" || in.Machine == "" || why == "" {
|
||||
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
|
||||
}
|
||||
key := runKeyOf(in.Candidate, in.Machine)
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, err
|
||||
}
|
||||
if !ok {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
|
||||
}
|
||||
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
|
||||
}
|
||||
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
|
||||
answer := meshwants.CompositionsDropAnswer{Run: run}
|
||||
answer.Run.Key = key
|
||||
if run.Ended == nil {
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
ids := make([]string, 0, len(run.Readers))
|
||||
for _, reader := range run.Readers {
|
||||
ids = append(ids, reader.ID)
|
||||
}
|
||||
c.leave(ids) // their waits say no end of their own
|
||||
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
|
||||
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
|
||||
return answer, err
|
||||
}
|
||||
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
|
||||
for _, reader := range run.Readers {
|
||||
if reader.Ended != nil && !reader.Rerun {
|
||||
continue // it ended passed already and keeps its verdict (A-470r)
|
||||
}
|
||||
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = run.Ask
|
||||
}
|
||||
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
|
||||
answer.Ended = append(answer.Ended, reader)
|
||||
}
|
||||
}
|
||||
// The dropped run's key goes while it holds that run: a run started on it since stands.
|
||||
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
|
||||
r.heard()
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
|
||||
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsInput
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
}
|
||||
return r.compositions(in)
|
||||
},
|
||||
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsDropInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return c.drop(r, record, caller, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
|
||||
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
call := func(verb string, args map[string]any, names ...string) (any, error) {
|
||||
in := map[string]any{}
|
||||
for _, n := range names {
|
||||
if v := arg(args, n); v != "" {
|
||||
in[n] = v
|
||||
}
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
|
||||
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositions, args, "candidate", "machine")
|
||||
}},
|
||||
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
|
||||
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
|
||||
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
|
||||
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
|
||||
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
|
||||
ends := newSaidEvents()
|
||||
var endIDs []string // the message id of each end said
|
||||
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
|
||||
endIDs = append(endIDs, id)
|
||||
return ends.say(event, body, id)
|
||||
}}
|
||||
moves := 0
|
||||
move := func(meshwants.CompositionRun) { moves++ }
|
||||
r.moved.Store(&move)
|
||||
for _, m := range []string{"n1", "n2"} {
|
||||
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"ask-p", "ask-q"} {
|
||||
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
|
||||
c.Ended, c.Rerun = &now, id == "ask-q"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
var recorded []meshhandactsclient.Request
|
||||
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
|
||||
verbs := compositionVerbs(r, c, record)
|
||||
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
|
||||
read, _ := out.(meshwants.CompositionsAnswer)
|
||||
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
|
||||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
|
||||
t.Fatalf("compositions = %+v %v", out, err)
|
||||
}
|
||||
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
|
||||
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
|
||||
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
|
||||
}
|
||||
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
|
||||
dropped, _ := out.(meshwants.CompositionsDropAnswer)
|
||||
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
|
||||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
|
||||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
|
||||
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
|
||||
}
|
||||
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
|
||||
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
|
||||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
|
||||
t.Errorf("the reader's end %+v", end)
|
||||
}
|
||||
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
|
||||
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
|
||||
}
|
||||
// The tools call the verbs with what they were given.
|
||||
var asked []string
|
||||
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
|
||||
})
|
||||
for _, tool := range tools {
|
||||
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
|
||||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
|
||||
t.Errorf("the tools asked %v", asked)
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
|
||||
@@ -23,6 +18,8 @@ type stepOutcome struct {
|
||||
Outcome string
|
||||
Said string
|
||||
Warnings []string
|
||||
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
|
||||
Shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// compositionSteps takes the composition steps and says their ends.
|
||||
@@ -30,49 +27,58 @@ type compositionSteps struct {
|
||||
node string
|
||||
// say says one event of the seat mesh-wants (wantsPublisher).
|
||||
say publisher
|
||||
// run runs one step and answers what it came to.
|
||||
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
|
||||
asker func() string
|
||||
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
|
||||
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
// validate runs one validate step and resumeValidate judges one taken up at the start (validatestep.go, design 47 H §745);
|
||||
// nil takes no validate step.
|
||||
validate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
resumeValidate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
|
||||
// writes none.
|
||||
ended func(id string, run meshwants.StepRun, at time.Time)
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
|
||||
ticks func(d time.Duration) (<-chan time.Time, func())
|
||||
}
|
||||
|
||||
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
|
||||
// accepted kind step, read from the seat list when it is said, since that subject names no asker (design 47 G A-294, which
|
||||
// corrects A-267). Empty while the seat list holds no definition naming exactly one.
|
||||
func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
|
||||
return func() string {
|
||||
d, ok := definition()
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
var one string
|
||||
for _, m := range d.Accepts {
|
||||
if m.Name != meshwants.AcceptStep || m.Callers == nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range m.Callers.List {
|
||||
name, isSeat := strings.CutPrefix(e, seatdef.CallerSeatPrefix)
|
||||
if !isSeat || (one != "" && one != name) {
|
||||
return ""
|
||||
}
|
||||
one = name
|
||||
}
|
||||
}
|
||||
return one
|
||||
}
|
||||
// taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not
|
||||
// pulled yet are kept there too (H-A-404).
|
||||
taken takenStore
|
||||
takenBound time.Duration
|
||||
// inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777).
|
||||
inProgress func() time.Duration
|
||||
mu sync.Mutex
|
||||
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is
|
||||
// dropped (the cause an *askDrop).
|
||||
stops map[string]context.CancelCauseFunc
|
||||
// held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said.
|
||||
held map[string][]queueMsg
|
||||
}
|
||||
|
||||
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
|
||||
const progressEvery = time.Minute
|
||||
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
|
||||
// composition runs it waits on (§723, §729).
|
||||
type stepPhase struct {
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// setShared names the runs the step ctx runs waits on, when it runs one.
|
||||
func setShared(ctx context.Context, shared []meshwants.Shared) {
|
||||
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
||||
p.mu.Lock()
|
||||
p.shared = shared
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *stepPhase) waitsOn() []meshwants.Shared {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.shared
|
||||
}
|
||||
|
||||
type phaseKey struct{}
|
||||
@@ -92,37 +98,133 @@ func (p *stepPhase) get() string {
|
||||
return p.phase
|
||||
}
|
||||
|
||||
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition step, or an ask with no
|
||||
// id, is left with a log line; one taken is run in its own goroutine and its end said.
|
||||
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
|
||||
var run meshwants.StepRun
|
||||
if err := json.Unmarshal(data, &run); err != nil || id == "" || run.Step != meshwants.StepRunStepComposition {
|
||||
c.logf("a step ask that is no composition step with an id is left: %v", err)
|
||||
return false
|
||||
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
|
||||
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
|
||||
switch {
|
||||
case step == meshwants.StepRunStepComposition && resumed:
|
||||
return c.resume
|
||||
case step == meshwants.StepRunStepComposition:
|
||||
return c.run
|
||||
case step == meshwants.StepRunStepValidate && resumed:
|
||||
return c.resumeValidate
|
||||
case step == meshwants.StepRunStepValidate:
|
||||
return c.validate
|
||||
}
|
||||
go c.runOne(ctx, id, run)
|
||||
return true
|
||||
return nil
|
||||
}
|
||||
|
||||
// runOne runs one step and says its end.
|
||||
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
|
||||
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
|
||||
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
|
||||
// says nothing: the runner ended it already.
|
||||
func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
|
||||
started := c.now().UTC()
|
||||
ctx, stopWait := context.WithCancelCause(parent)
|
||||
c.mu.Lock()
|
||||
if c.stops == nil {
|
||||
c.stops = map[string]context.CancelCauseFunc{}
|
||||
}
|
||||
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
|
||||
c.mu.Unlock()
|
||||
stopWait(nil)
|
||||
return
|
||||
}
|
||||
c.stops[id] = stopWait
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
delete(c.stops, id)
|
||||
c.mu.Unlock()
|
||||
stopWait(nil)
|
||||
}()
|
||||
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
|
||||
stop := c.sayProgress(id, phase)
|
||||
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
stop := c.sayProgress(id, run.Asker, phase)
|
||||
stopInProgress := c.inProgressEvery(id)
|
||||
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
stopInProgress()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start
|
||||
return
|
||||
}
|
||||
var d *askDrop
|
||||
if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401)
|
||||
c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again)
|
||||
return
|
||||
}
|
||||
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
|
||||
c.settle(id, true) // the runner ended it already: its ask is over
|
||||
return
|
||||
}
|
||||
c.sayEnd(id, run, &started, o, again)
|
||||
}
|
||||
|
||||
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
|
||||
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
|
||||
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
|
||||
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
|
||||
ended := c.now().UTC()
|
||||
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
|
||||
if started == nil {
|
||||
started = &ended
|
||||
}
|
||||
step := meshwants.StepEndedStepComposition
|
||||
if run.Step == meshwants.StepRunStepValidate {
|
||||
step = meshwants.StepEndedStepValidate
|
||||
}
|
||||
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: step, Module: run.Module,
|
||||
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
|
||||
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
|
||||
c.logf("saying the end of the composition step %s: %v", id, err)
|
||||
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
|
||||
msgID := id
|
||||
if again != "" {
|
||||
msgID = id + "." + again
|
||||
}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
|
||||
c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err)
|
||||
c.settle(id, false)
|
||||
return
|
||||
}
|
||||
c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777)
|
||||
// One journal line per ask answered, in the one place it is answered (design 47 H2 §792).
|
||||
c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome)
|
||||
if c.ended != nil {
|
||||
c.ended(id, run, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
|
||||
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
|
||||
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
|
||||
return func(run meshwants.CompositionRun) {
|
||||
for _, reader := range run.Readers {
|
||||
if !reader.Rerun || c.running(reader.ID) {
|
||||
continue
|
||||
}
|
||||
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// running is whether the step asked under id runs here.
|
||||
func (c *compositionSteps) running(id string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.stops[id] != nil
|
||||
}
|
||||
|
||||
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
|
||||
func (c *compositionSteps) leave(ids []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
if stop := c.stops[id]; stop != nil {
|
||||
stop(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
|
||||
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
|
||||
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
|
||||
func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) {
|
||||
ticks := c.ticks
|
||||
if ticks == nil {
|
||||
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
|
||||
@@ -139,7 +241,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
|
||||
return
|
||||
case <-tick:
|
||||
at := c.now().UTC()
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()}
|
||||
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
c.logf("saying the progress of the composition step %s: %v", id, err)
|
||||
}
|
||||
@@ -148,21 +250,3 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
|
||||
}()
|
||||
return func() { close(done); <-ended }
|
||||
}
|
||||
|
||||
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
|
||||
func stepAskID(m *nats.Msg) string {
|
||||
if id := m.Header.Get("x-event-id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return m.Header.Get(nats.MsgIdHdr)
|
||||
}
|
||||
|
||||
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
|
||||
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
|
||||
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Drain() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
+186
-12
@@ -5,6 +5,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -105,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
|
||||
from := b.journal.mark()
|
||||
b.start(t)
|
||||
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
|
||||
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
|
||||
|
||||
select {
|
||||
@@ -113,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
|
||||
putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
|
||||
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
|
||||
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
||||
msg.Header.Set("x-event-id", "step-ask-1")
|
||||
msg.Data = step
|
||||
if err := nc.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
|
||||
// its end with the composer's verdict: n1 composes, n2 does not.
|
||||
@@ -146,6 +141,9 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
if err != nil || len(req.Machines) != 1 {
|
||||
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
||||
}
|
||||
if len(req.Seats) == 0 { // the revision names no definition: the request carries the trunk seat list's (A-467)
|
||||
t.Errorf("the request of %s carries no seat definition", a.ID)
|
||||
}
|
||||
node := req.Machines[0]
|
||||
programs[node] = a.Run.Program.Commit
|
||||
m := composer.Machine{Composes: true}
|
||||
@@ -165,7 +163,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
t.Fatal("the composition step said no end")
|
||||
}
|
||||
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
|
||||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
|
||||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
|
||||
e.Warnings[1] != "n2: the composer names no culprits" {
|
||||
t.Fatalf("the step ended %+v", e)
|
||||
}
|
||||
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
|
||||
@@ -177,10 +176,26 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
t.Error("the machine away was asked for its input")
|
||||
}
|
||||
}
|
||||
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
|
||||
if len(keys) != 0 {
|
||||
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
|
||||
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
|
||||
inputs := b.buckets[buckets.Inputs()]
|
||||
w, err := inputs.WatchAll(b.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
keys, _ := inputs.Keys(b.ctx)
|
||||
if len(keys) == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-w.Updates():
|
||||
case <-deadline:
|
||||
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
|
||||
}
|
||||
}
|
||||
queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777)
|
||||
}
|
||||
|
||||
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
|
||||
@@ -222,3 +237,162 @@ func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// Design 47 H §745 (acceptance test 117, the assigner's half; H-A-108), started whole on the bus: the assigner serves the step
|
||||
// validate on the seat mesh-wants. A take-in whose manifest names an unknown field fails it `manifest-refused` naming it, the
|
||||
// validator being the revision's own entry of the module holding the seat mesh-catalog, run on the Go build seat over the
|
||||
// revision's manifests and the previous revision's; a take-in that changes a manifest with no problem passes it; a take-in whose
|
||||
// entries make a cycle fails it `cycle` naming the cycle, with no validator asked.
|
||||
func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const commit = "2222222222222222222222222222222222222222"
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commit, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
|
||||
entry := func(module, digest string, manifest map[string]any, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
||||
manifest["module"] = module
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: manifest, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.rev[trunkBranch] = 9
|
||||
b.cat.at = map[string]map[int64][]meshcatalog.CatalogEntry{trunkBranch: {
|
||||
6: {indexer, entry("web", "d-web-1", map[string]any{"version": "1"})},
|
||||
7: {indexer, entry("web", "d-web-2", map[string]any{"version": "2", "colour": "red"})},
|
||||
8: {indexer, entry("web", "d-web-3", map[string]any{"version": "3"})},
|
||||
9: {indexer, entry("a", "d-a", map[string]any{}, meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}),
|
||||
entry("b", "d-b", map[string]any{})},
|
||||
}}
|
||||
b.cat.mu.Unlock()
|
||||
ended := make(chan meshwants.StepEnded, 4)
|
||||
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data, &e)
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ends.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
// Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken)
|
||||
// and ended after its start.
|
||||
taken := make(chan *nats.Msg, 8)
|
||||
takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = takens.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a",
|
||||
Branch: trunkBranch, Revision: 9, Commit: commit})
|
||||
from := b.journal.mark()
|
||||
b.start(t)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second)
|
||||
select {
|
||||
case m := <-taken:
|
||||
var at meshwants.AskTakenEvent
|
||||
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") {
|
||||
t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped")
|
||||
}
|
||||
select {
|
||||
case e := <-ended:
|
||||
if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
|
||||
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the validate ask put while the assigner was stopped said no end")
|
||||
}
|
||||
select {
|
||||
case <-b.seatListRead:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := func(revision int64, module string, run func()) meshwants.StepEnded {
|
||||
t.Helper()
|
||||
id := fmt.Sprintf("validate-%d", revision)
|
||||
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
|
||||
Branch: trunkBranch, Revision: revision, Commit: commit})
|
||||
var asked meshwants.StepRun
|
||||
_ = json.Unmarshal(step, &asked)
|
||||
putStep(b.ctx, t, b.js, id, asked)
|
||||
if run != nil {
|
||||
run()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case e := <-ended:
|
||||
if e.ID == id {
|
||||
return e
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("the validate step of revision %d said no end", revision)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The stand-in Go build seat runs the validator: it refuses a manifest naming the field colour, and accepts every other.
|
||||
validator := func() {
|
||||
t.Helper()
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("no validator run is asked: %v", err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil || a.Run.Program.Repository != "novox/mesh-delivery" || a.Run.Program.Commit != commit {
|
||||
t.Fatalf("the validator's run ask %s: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase {
|
||||
t.Fatalf("the validator's request: %+v %v", req, err)
|
||||
}
|
||||
// The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443).
|
||||
if list, digest := seatListBytes(); req.SeatList != digest {
|
||||
t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest)
|
||||
} else if got, _ := json.Marshal(req.Seats); string(got) != string(list) {
|
||||
t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list))
|
||||
}
|
||||
v := composer.Verdict{V: composer.Version}
|
||||
for _, c := range req.Cases {
|
||||
reads := map[string]composer.ManifestRead{}
|
||||
for module, m := range c.Manifests {
|
||||
reads[module] = composer.ManifestRead{Accepted: true}
|
||||
if strings.Contains(string(m.Raw), "colour") {
|
||||
reads[module] = composer.ManifestRead{Refused: `unknown field "colour"`}
|
||||
}
|
||||
}
|
||||
v.Cases = append(v.Cases, composer.CaseVerdict{Name: c.Name, Manifests: reads})
|
||||
}
|
||||
out, _ := json.Marshal(v)
|
||||
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(out), At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if e := ask(7, "web", validator); e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(e.Said, reasonManifestRefused+": ") || !strings.Contains(e.Said, `web: unknown field "colour"`) {
|
||||
t.Errorf("the validate step of a manifest naming an unknown field ended %+v", e)
|
||||
}
|
||||
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
|
||||
}
|
||||
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
|
||||
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
|
||||
}
|
||||
queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777)
|
||||
}
|
||||
|
||||
+399
-26
@@ -3,14 +3,115 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled.
|
||||
type fakeMsg struct {
|
||||
data []byte
|
||||
h nats.Header
|
||||
mu sync.Mutex
|
||||
acks, naks, terms, inProgresss int
|
||||
}
|
||||
|
||||
func (m *fakeMsg) Data() []byte { return m.data }
|
||||
func (m *fakeMsg) Headers() nats.Header { return m.h }
|
||||
func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil }
|
||||
func (m *fakeMsg) Ack() error { return m.count(&m.acks) }
|
||||
func (m *fakeMsg) Nak() error { return m.count(&m.naks) }
|
||||
func (m *fakeMsg) Term() error { return m.count(&m.terms) }
|
||||
func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) }
|
||||
func (m *fakeMsg) settled() (acks, naks, terms, progress int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.acks, m.naks, m.terms, m.inProgresss
|
||||
}
|
||||
|
||||
// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set.
|
||||
type fakeTaken struct {
|
||||
mu sync.Mutex
|
||||
values map[string][]byte
|
||||
ttls map[string]time.Duration
|
||||
refuse error
|
||||
}
|
||||
|
||||
func newFakeTaken() *fakeTaken {
|
||||
return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}}
|
||||
}
|
||||
|
||||
func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.refuse != nil {
|
||||
return false, f.refuse
|
||||
}
|
||||
key := seat + "." + ask
|
||||
if _, ok := f.values[key]; ok {
|
||||
return false, nil
|
||||
}
|
||||
f.values[key], f.ttls[key] = []byte(`{}`), bound
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
v, ok := f.values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: key, Value: v}, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) {
|
||||
raw, err := json.Marshal(value)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.values[key], f.ttls[key] = raw, ttl
|
||||
return 1, err
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Delete(key string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Keys() ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none.
|
||||
func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg {
|
||||
if c.taken == nil {
|
||||
c.taken = newFakeTaken()
|
||||
}
|
||||
m := &fakeMsg{data: body, h: nats.Header{}}
|
||||
if id != "" {
|
||||
m.h.Set("x-event-id", id)
|
||||
}
|
||||
c.pulled(context.Background(), m)
|
||||
return m
|
||||
}
|
||||
|
||||
// saidEvents keeps what a publisher says.
|
||||
type saidEvents struct {
|
||||
mu sync.Mutex
|
||||
@@ -37,23 +138,34 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
said := newSaidEvents()
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
var asked meshwants.StepRun
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
|
||||
var lines []string
|
||||
var linesMu sync.Mutex
|
||||
logf := func(f string, a ...any) {
|
||||
linesMu.Lock()
|
||||
lines = append(lines, fmt.Sprintf(f, a...))
|
||||
linesMu.Unlock()
|
||||
}
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf,
|
||||
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
|
||||
asked = run
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
|
||||
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
|
||||
}}
|
||||
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
|
||||
if c.take(context.Background(), "ask-0", other) {
|
||||
if m := pullOne(c, "ask-0", other); m.terms != 1 {
|
||||
t.Fatal("a step ask of another step is taken")
|
||||
}
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
|
||||
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
|
||||
if c.take(context.Background(), "", body) {
|
||||
if m := pullOne(c, "", body); m.terms != 1 {
|
||||
t.Fatal("a step ask with no id is taken")
|
||||
}
|
||||
if !c.take(context.Background(), "ask-1", body) {
|
||||
if m := pullOne(c, "ask-1", body); m.terms != 0 {
|
||||
t.Fatal("the composition step is not taken")
|
||||
}
|
||||
if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777)
|
||||
t.Fatalf("said %s first, want ask-taken", e)
|
||||
}
|
||||
select {
|
||||
case <-said.heard:
|
||||
case <-time.After(5 * time.Second):
|
||||
@@ -67,9 +179,215 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
|
||||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
|
||||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
|
||||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
|
||||
t.Fatalf("step-ended %+v", end)
|
||||
}
|
||||
// Design 47 H2 §792: the ask answered is one journal line, said where its end is said.
|
||||
linesMu.Lock()
|
||||
answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") })
|
||||
linesMu.Unlock()
|
||||
if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) {
|
||||
t.Errorf("the journal says %q, want one answered line", answered)
|
||||
}
|
||||
// Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at
|
||||
// the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop.
|
||||
t.Run("work queue", queueCases)
|
||||
}
|
||||
|
||||
func queueCases(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4})
|
||||
type bed struct {
|
||||
c *compositionSteps
|
||||
said *saidEvents
|
||||
taken *fakeTaken
|
||||
runs chan string
|
||||
release chan struct{}
|
||||
tick chan time.Time
|
||||
}
|
||||
newBed := func() *bed {
|
||||
b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)}
|
||||
b.said.heard = make(chan string, 16)
|
||||
run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome {
|
||||
b.runs <- id
|
||||
select {
|
||||
case <-b.release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}
|
||||
b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute,
|
||||
validate: run, inProgress: func() time.Duration { return 20 * time.Second },
|
||||
ticks: func(d time.Duration) (<-chan time.Time, func()) {
|
||||
if d == 20*time.Second {
|
||||
return b.tick, func() {}
|
||||
}
|
||||
return make(chan time.Time), func() {}
|
||||
}}
|
||||
return b
|
||||
}
|
||||
ended := func(t *testing.T, b *bed) meshwants.StepEnded {
|
||||
t.Helper()
|
||||
for {
|
||||
select {
|
||||
case e := <-b.said.heard:
|
||||
if e == meshwants.EventStepEnded+".n1" {
|
||||
b.said.mu.Lock()
|
||||
defer b.said.mu.Unlock()
|
||||
return b.said.said[e].(meshwants.StepEnded)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no step-ended is said")
|
||||
}
|
||||
}
|
||||
}
|
||||
settledAt := func(t *testing.T, m *fakeMsg, acks int) {
|
||||
t.Helper()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
if a, _, _, _ := m.settled(); a == acks {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks)
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) {
|
||||
b := newBed()
|
||||
m := pullOne(b.c, "ask-1", body)
|
||||
if id := <-b.runs; id != "ask-1" {
|
||||
t.Fatalf("ran %s", id)
|
||||
}
|
||||
b.said.mu.Lock()
|
||||
taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent)
|
||||
takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"]
|
||||
b.said.mu.Unlock()
|
||||
if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute {
|
||||
t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls)
|
||||
}
|
||||
b.tick <- at
|
||||
b.tick <- at
|
||||
// handed again while it runs: held for the run, which answers both at its one end
|
||||
again := pullOne(b.c, "ask-1", body)
|
||||
if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 {
|
||||
t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p)
|
||||
}
|
||||
close(b.release)
|
||||
if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
settledAt(t, again, 1)
|
||||
select {
|
||||
case id := <-b.runs:
|
||||
t.Fatalf("the ask handed again ran again (%s)", id)
|
||||
default:
|
||||
}
|
||||
})
|
||||
t.Run("taken before is run again, said no ask-taken", func(t *testing.T) {
|
||||
b := newBed()
|
||||
_, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute)
|
||||
close(b.release)
|
||||
m := pullOne(b.c, "ask-2", body)
|
||||
<-b.runs
|
||||
ended(t, b)
|
||||
settledAt(t, m, 1)
|
||||
b.said.mu.Lock()
|
||||
_, said := b.said.said[meshwants.EventAskTaken+".ask-2"]
|
||||
b.said.mu.Unlock()
|
||||
if said {
|
||||
t.Fatal("ask-taken is said again for an ask taken before")
|
||||
}
|
||||
})
|
||||
t.Run("a refused take is handed again", func(t *testing.T) {
|
||||
b := newBed()
|
||||
b.taken.refuse = errors.New("no bus")
|
||||
m := pullOne(b.c, "ask-3", body)
|
||||
if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 {
|
||||
t.Fatalf("nakked %d, ran %d", n, len(b.runs))
|
||||
}
|
||||
})
|
||||
t.Run("dropped while it runs", func(t *testing.T) {
|
||||
b := newBed()
|
||||
m := pullOne(b.c, "ask-4", body)
|
||||
<-b.runs
|
||||
if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil {
|
||||
t.Fatal("a drop with no why is taken")
|
||||
}
|
||||
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"})
|
||||
if err != nil || a["held"] != true {
|
||||
t.Fatalf("drop answered %v, %v", a, err)
|
||||
}
|
||||
e := ended(t, b)
|
||||
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
})
|
||||
t.Run("dropped before it was pulled", func(t *testing.T) {
|
||||
b := newBed()
|
||||
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"})
|
||||
if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute {
|
||||
t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls)
|
||||
}
|
||||
m := pullOne(b.c, "ask-5", body)
|
||||
e := ended(t, b)
|
||||
if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
if len(b.runs) != 0 {
|
||||
t.Fatal("a dropped ask ran")
|
||||
}
|
||||
if v, _ := b.taken.Get("dropped.ask-5"); v != nil {
|
||||
t.Fatal("the drop's key stays once its ask ended")
|
||||
}
|
||||
})
|
||||
t.Run("the asks tools", func(t *testing.T) {
|
||||
taken := newFakeTaken()
|
||||
_, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute)
|
||||
_, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute)
|
||||
waiting := func(seat string) (stdio.Waiting, error) {
|
||||
if seat != meshwants.Seat {
|
||||
t.Errorf("the waiting asks of %s are read", seat)
|
||||
}
|
||||
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3},
|
||||
{ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil
|
||||
}
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
if strings.HasSuffix(key, "."+meshwants.VerbDrop) {
|
||||
return json.RawMessage(`{"held":false}`), nil
|
||||
}
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
tools := map[string]stdio.Tool{}
|
||||
for _, tool := range asksTools(waiting, taken, ask) {
|
||||
tools[tool.Name] = tool
|
||||
}
|
||||
out, err := tools[asksTool].Run(map[string]any{})
|
||||
a, _ := out.(map[string]any)
|
||||
w, _ := a["waiting"].([]map[string]any)
|
||||
d, _ := a["dropped"].([]map[string]any)
|
||||
if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 ||
|
||||
d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false {
|
||||
t.Fatalf("assigner_asks answered %+v, %v", out, err)
|
||||
}
|
||||
if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 {
|
||||
t.Fatalf("a drop with no why: %v, asked %v", err, asked)
|
||||
}
|
||||
out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"})
|
||||
if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") ||
|
||||
!strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) {
|
||||
t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
|
||||
@@ -80,23 +398,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
stopped := make(chan struct{})
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
release := make(chan struct{})
|
||||
// A-294: the asker is the one caller mesh-wants' definition lists for step, read from the seat list; none while unread
|
||||
// or while it lists more than one.
|
||||
runner := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
|
||||
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}}
|
||||
two := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
|
||||
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner", "seat:mesh-catalog"}}}}}
|
||||
for _, d := range []struct {
|
||||
def seatdef.Definition
|
||||
read bool
|
||||
want string
|
||||
}{{runner, true, "mesh-pipeline-runner"}, {runner, false, ""}, {two, true, ""}} {
|
||||
if got := stepAskerOf(func() (seatdef.Definition, bool) { return d.def, d.read })(); got != d.want {
|
||||
t.Errorf("the asker of %+v (read %v) = %q, want %q", d.def.Accepts, d.read, got, d.want)
|
||||
}
|
||||
}
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
|
||||
asker: stepAskerOf(func() (seatdef.Definition, bool) { return runner, true }),
|
||||
ticks: func(d time.Duration) (<-chan time.Time, func()) {
|
||||
if d != time.Minute {
|
||||
t.Errorf("the progress ticks every %s, want a minute", d)
|
||||
@@ -105,13 +407,16 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
},
|
||||
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
|
||||
<-release
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main"})
|
||||
if !c.take(context.Background(), "ask-1", body) {
|
||||
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
|
||||
if m := pullOne(c, "ask-1", body); m.terms != 0 {
|
||||
t.Fatal("the composition step is not taken")
|
||||
}
|
||||
<-said.heard // ask-taken, at the pull (H2 §777)
|
||||
progress := meshwants.EventStepProgress + ".n1"
|
||||
for range 2 {
|
||||
tick <- at
|
||||
@@ -122,7 +427,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
said.mu.Lock()
|
||||
p, ok := said.said[progress].(meshwants.StepProgressEvent)
|
||||
said.mu.Unlock()
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
|
||||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
|
||||
t.Fatalf("step-progress %+v", p)
|
||||
}
|
||||
close(release)
|
||||
@@ -135,3 +441,70 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
t.Fatal("the ticker is not stopped when the step has ended")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
|
||||
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
|
||||
// reader is left", so a later step starts a new run.
|
||||
func TestASupersededStepLeavesItsRuns(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
step := func(pipeline string) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
|
||||
}
|
||||
said := newSaidEvents()
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
|
||||
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
|
||||
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
r.await(ctx, []string{j.key})
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
|
||||
for _, p := range []string{"main.3.a", "main.3.b"} {
|
||||
if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 {
|
||||
t.Fatal("not taken")
|
||||
}
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
for {
|
||||
wake := r.wake()
|
||||
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
|
||||
break
|
||||
}
|
||||
<-wake
|
||||
}
|
||||
ids, err := r.leave("main.3.a", c.leave)
|
||||
run, _, _, _ := r.read(key)
|
||||
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
|
||||
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
|
||||
}
|
||||
_, err = r.leave("main.3.b", c.leave)
|
||||
run, _, _, _ = r.read(key)
|
||||
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
|
||||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
|
||||
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
n := len(c.stops)
|
||||
c.mu.Unlock()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("the steps that left still wait")
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
said.mu.Lock()
|
||||
defer said.mu.Unlock()
|
||||
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
|
||||
t.Errorf("a step that left says its end: %v", said.said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,451 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A delivery is waited for mesh-wide (design 47 G §376, §389, A-189, A-246). At its start the assigner reads whole, from their first
|
||||
// message, the streams of the catalog seat's version-released and of its own version-delivered (mesh-sdk process.FromFirst, A-233),
|
||||
// and the watchman's machine-away and machine-back, never its state; then it hears them. A released version of a module is said
|
||||
// version-delivered on the seat mesh-wants once, when every present machine placing it on the branch it is released from runs, healthy,
|
||||
// the want the assigner wrote naming that version; a version no present machine places is not delivered. A dependent whose want
|
||||
// changes fingerprint waits, by an edge of kind 2, 4 or 6, for its dependency's newest released version not yet said delivered, on
|
||||
// the branch it is released from; on another branch no version is released and nothing waits. The wait is said wait-held, step
|
||||
// delivery, bound 60 minutes from the version's release, kept in the state delivery-waits under the dependent's name, and said
|
||||
// wait-ended when the version is delivered (ran) or released by hand (assigner_release, released).
|
||||
|
||||
// deliveryWaitBound is a dependent's delivery wait's bound, from its dependency's version-released.
|
||||
const deliveryWaitBound = 60 * time.Minute
|
||||
|
||||
// Wait-ended's how of a delivery wait.
|
||||
const (
|
||||
deliveryWaitRan = "ran"
|
||||
deliveryWaitReleased = "released"
|
||||
)
|
||||
|
||||
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
|
||||
type releaseHeard struct {
|
||||
Version string
|
||||
Parts map[string]string
|
||||
Source int64
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// newestOf is the newest version of module released, the last heard of it in the stream's order.
|
||||
func (r *releasedVersions) newestOf(module string) (releaseHeard, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
h, ok := r.newest[module]
|
||||
return h, ok
|
||||
}
|
||||
|
||||
// deliveredVersions are the version-delivered said, the assigner's own, by module and version: each is said once.
|
||||
type deliveredVersions struct {
|
||||
mu sync.Mutex
|
||||
said map[string]bool
|
||||
}
|
||||
|
||||
func (d *deliveredVersions) key(module, version string) string { return module + "@" + version }
|
||||
|
||||
// put takes one version-delivered in.
|
||||
func (d *deliveredVersions) put(data []byte) error {
|
||||
var e meshwants.VersionDeliveredEvent
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return fmt.Errorf("a version-delivered that is none: %w", err)
|
||||
}
|
||||
if e.Module == "" || e.Version == "" {
|
||||
return fmt.Errorf("a version-delivered naming no module or version: %s", data)
|
||||
}
|
||||
d.mark(e.Module, e.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deliveredVersions) mark(module, version string) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.said == nil {
|
||||
d.said = map[string]bool{}
|
||||
}
|
||||
d.said[d.key(module, version)] = true
|
||||
}
|
||||
|
||||
// has says whether version-delivered of module's version was said.
|
||||
func (d *deliveredVersions) has(module, version string) bool {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.said[d.key(module, version)]
|
||||
}
|
||||
|
||||
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
|
||||
func presenceOf(data []byte) (string, time.Time, error) {
|
||||
var e wd.MachineAwayEvent // machine-back has the same fields
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
if e.Machine == "" {
|
||||
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
|
||||
}
|
||||
return e.Machine, e.At, nil
|
||||
}
|
||||
|
||||
// hearDelivery reads whole, then follows, the four streams a delivery is judged from: version-released, version-delivered,
|
||||
// machine-away and machine-back. A stream not heard is said as the assigner's finding, and what it carries is not known until
|
||||
// the assigner starts again.
|
||||
func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
|
||||
acct := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
nh := notHeardWriter(findings, logf)
|
||||
for _, s := range []struct {
|
||||
stream, words string
|
||||
take func([]byte) error
|
||||
}{
|
||||
{meshcatalog.StreamVersionReleased, "no want names a released version and no delivery is judged", w.released.put},
|
||||
{meshwants.StreamVersionDelivered, "a version said delivered before is said again once it is judged", w.delivered.put},
|
||||
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, true, at)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, false, at)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
take := s.take
|
||||
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
|
||||
if err := take(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf, process.FromFirst())
|
||||
if err != nil {
|
||||
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
|
||||
continue
|
||||
}
|
||||
nh.Heard(s.stream)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// releaseBranch is the branch module's versions are released from: the first watched branch of its registration (§371).
|
||||
func (p *passer) releaseBranch(module string) string {
|
||||
if p.ends.following() {
|
||||
return p.ends.branchOf(module, "")
|
||||
}
|
||||
return p.w.trunkName()
|
||||
}
|
||||
|
||||
// placing is every machine placing module on the branch its versions are released from, sorted.
|
||||
func (p *passer) placing(module string) []string {
|
||||
branch := p.releaseBranch(module)
|
||||
var out []string
|
||||
for node, placed := range p.w.placedBranches()[module] {
|
||||
if p.ends.following() {
|
||||
placed = p.ends.branchOf(module, placed)
|
||||
} else {
|
||||
placed = p.w.trunkName() // the trunk-only form: every placement runs the trunk's
|
||||
}
|
||||
if placed == branch {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryStanding is where module's version stands: the present placing machines not yet running it healthy as its want names
|
||||
// it, and the placing machines away.
|
||||
func (p *passer) deliveryStanding(module, version string) (notRunning, away []string, placing int) {
|
||||
gone := p.w.awayNow()
|
||||
n := assign.Nodes{Running: p.w.runningNow()}
|
||||
wanted := p.w.wantsOf()[module]
|
||||
nodes := p.placing(module)
|
||||
for _, node := range nodes {
|
||||
if gone[node] {
|
||||
away = append(away, node)
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[node]
|
||||
if !ok || want.Removed || want.Version != version || !n.RunsHealthy(node, module, want.Fp) {
|
||||
notRunning = append(notRunning, node)
|
||||
}
|
||||
}
|
||||
return notRunning, away, len(nodes)
|
||||
}
|
||||
|
||||
// judgeDelivered says version-delivered of each module's newest released version not said yet, once every present machine placing
|
||||
// it runs it healthy; judged again at each pass, which each running, machine-away, machine-back and placement change wakes.
|
||||
func (p *passer) judgeDelivered() {
|
||||
for _, module := range p.releasedModules() {
|
||||
rel, _ := p.w.released.newestOf(module)
|
||||
if p.w.delivered.has(module, rel.Version) {
|
||||
continue
|
||||
}
|
||||
notRunning, away, placing := p.deliveryStanding(module, rel.Version)
|
||||
if placing == 0 || len(away) == placing || len(notRunning) > 0 {
|
||||
continue
|
||||
}
|
||||
if err := p.sayDelivered(module, rel, away); err != nil {
|
||||
p.logf("version-delivered of %s %s is not said, and is judged again at the next pass: %v", module, rel.Version, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayDelivered says version-delivered of module's release rel, with the machines away, and keeps it said.
|
||||
func (p *passer) sayDelivered(module string, rel releaseHeard, away []string) error {
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.VersionDeliveredEvent{V: keys.Version, Module: module, Version: rel.Version, Parts: rel.Parts, Away: away, At: &at}
|
||||
if err := p.wantsSay(meshwants.EventVersionDelivered, body, fmt.Sprintf("%s.%s.%s.%s", meshwants.EventVersionDelivered, module, rel.Version, at.Format(time.RFC3339Nano))); err != nil {
|
||||
return err
|
||||
}
|
||||
p.w.delivered.mark(module, rel.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// releasedModules is every module a version-released was heard of, sorted.
|
||||
func (p *passer) releasedModules() []string {
|
||||
r := &p.w.released
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
out := make([]string, 0, len(r.newest))
|
||||
for m := range r.newest {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWait is what a dependent waits for: its dependency, the version and the machines not yet running it.
|
||||
type deliveryWait struct {
|
||||
dependency string
|
||||
rel releaseHeard
|
||||
notRunning []string
|
||||
away []string
|
||||
}
|
||||
|
||||
// waitsForDelivery says whether module, whose want on its nodes changes to fp on branch, waits for a dependency's delivery: by
|
||||
// each edge of kind 2, 4 or 6 into it in branch's catalog half, the dependency's newest released version on branch not yet said
|
||||
// delivered. A want that changes no fingerprint waits for nothing; on a branch no version is released from, nothing waits.
|
||||
func (p *passer) waitsForDelivery(module, fp, branch string, nodes []string) (deliveryWait, bool) {
|
||||
wanted := p.w.wantsOf()[module]
|
||||
changes := false
|
||||
for _, node := range nodes {
|
||||
if w, ok := wanted[node]; !ok || w.Removed || w.Fp != fp {
|
||||
changes = true
|
||||
}
|
||||
}
|
||||
if !changes || fp == "" {
|
||||
return deliveryWait{}, false
|
||||
}
|
||||
var deps []string
|
||||
for _, e := range p.w.edgesOf(branch, module) {
|
||||
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
|
||||
continue
|
||||
}
|
||||
dep, _, _ := strings.Cut(e.From, ":")
|
||||
if dep != "" && dep != module && !slices.Contains(deps, dep) {
|
||||
deps = append(deps, dep)
|
||||
}
|
||||
}
|
||||
sort.Strings(deps)
|
||||
for _, dep := range deps {
|
||||
if p.releaseBranch(dep) != branch {
|
||||
continue
|
||||
}
|
||||
rel, ok := p.w.released.newestOf(dep)
|
||||
if !ok || p.w.delivered.has(dep, rel.Version) {
|
||||
continue
|
||||
}
|
||||
notRunning, away, _ := p.deliveryStanding(dep, rel.Version)
|
||||
return deliveryWait{dependency: dep, rel: rel, notRunning: notRunning, away: away}, true
|
||||
}
|
||||
return deliveryWait{}, false
|
||||
}
|
||||
|
||||
// words is a delivery wait in plain words, as assigner_why_not and the status view say it.
|
||||
func (d deliveryWait) words() string {
|
||||
if len(d.notRunning) == 0 && len(d.away) > 0 {
|
||||
return fmt.Sprintf("waiting for %s to be back", strings.Join(d.away, ", "))
|
||||
}
|
||||
return fmt.Sprintf("waiting for delivery of %s %s on %s", d.dependency, d.rel.Version, strings.Join(d.notRunning, ", "))
|
||||
}
|
||||
|
||||
// deliveryWaitBody is the wait-held of module's delivery wait d.
|
||||
func deliveryWaitBody(module, fp string, d deliveryWait, since time.Time) meshwants.Wait {
|
||||
from := d.rel.At.UTC()
|
||||
if from.IsZero() {
|
||||
from = since.UTC()
|
||||
}
|
||||
return meshwants.Wait{Step: meshwants.WaitStepDelivery, Owner: meshwants.Seat, Module: module, Version: fp, Part: d.dependency,
|
||||
On: d.dependency + "@" + d.rel.Version, Machines: d.notRunning, Since: since.UTC(), ClockFrom: &from, Bound: deliveryWaitBound.String(),
|
||||
Words: d.words()}
|
||||
}
|
||||
|
||||
// deliveryWaiting is one delivery wait the state delivery-waits keeps, by its dependent's name: the wait-held said, and a release
|
||||
// by hand written by assigner_release.
|
||||
type deliveryWaiting struct {
|
||||
meshwants.Wait
|
||||
// Released is set by assigner_release: the wait ends released and the dependent's want is written.
|
||||
Released bool `json:"released,omitempty"`
|
||||
}
|
||||
|
||||
// saidDeliveryWaits is every delivery wait the state delivery-waits holds, by module; a value that does not decode is said and left.
|
||||
func (p *passer) saidDeliveryWaits() map[string]deliveryWaiting {
|
||||
out := map[string]deliveryWaiting{}
|
||||
for key, value := range p.w.read(buckets.DeliveryWaits()) {
|
||||
var w deliveryWaiting
|
||||
if err := json.Unmarshal(value, &w); err != nil || w.Module == "" {
|
||||
p.logf("%s %s holds no delivery wait, left: %v", buckets.DeliveryWaits(), key, err)
|
||||
continue
|
||||
}
|
||||
out[w.Module] = w
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// holdForDelivery says whether module's want, changing to fp on branch, is held for a dependency's delivery this pass: it says
|
||||
// wait-held once and keeps it; a wait released by hand holds nothing; a wait that no longer holds is ended ran.
|
||||
func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string, nodes []string) bool {
|
||||
if p.kvs[buckets.DeliveryWaits()] == nil {
|
||||
return false
|
||||
}
|
||||
key := keys.EscapeName(module)
|
||||
said, was := p.saidDeliveryWaits()[module]
|
||||
d, waits := p.waitsForDelivery(module, fp, branch, nodes)
|
||||
switch {
|
||||
case was && said.Released:
|
||||
p.endDeliveryWait(ctx, key, said, deliveryWaitReleased) // the want is written now, so it changes nothing at the next pass
|
||||
return false
|
||||
case !waits:
|
||||
if was {
|
||||
p.endDeliveryWait(ctx, key, said, deliveryWaitRan)
|
||||
}
|
||||
return false
|
||||
}
|
||||
body := deliveryWaitBody(module, fp, d, p.clock())
|
||||
if was && said.On == body.On && said.Version == fp && slices.Equal(said.Machines, body.Machines) {
|
||||
return true
|
||||
}
|
||||
if was {
|
||||
body.Since = said.Since
|
||||
}
|
||||
id := fmt.Sprintf("%s.delivery.%s.%s.%s", key, fp, body.On, strings.Join(body.Machines, ","))
|
||||
if err := p.wantsSay(meshwants.EventWaitHeld, body, id); err != nil {
|
||||
p.logf("the delivery wait of %s is not said, and is said at the next pass: %v", module, err)
|
||||
return true
|
||||
}
|
||||
kept := deliveryWaiting{Wait: body}
|
||||
p.put(ctx, buckets.DeliveryWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
|
||||
return true
|
||||
}
|
||||
|
||||
// endDeliveryWait says wait-ended of a delivery wait with how and removes it from the state.
|
||||
func (p *passer) endDeliveryWait(ctx context.Context, key string, w deliveryWaiting, how string) {
|
||||
body := w.Wait
|
||||
body.How = how
|
||||
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+".delivery."+body.Version+"."+how); err != nil {
|
||||
p.logf("the end of the delivery wait of %s is not said, and is said at the next pass: %v", w.Module, err)
|
||||
return
|
||||
}
|
||||
p.remove(ctx, buckets.DeliveryWaits(), key)
|
||||
}
|
||||
|
||||
// againAsk is one ask of assigner_delivered_again, kept in the state delivered-again under its module (A-250): the version to say
|
||||
// delivered again, who asked and why, and, while it does not hold, why not.
|
||||
type againAsk struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Waits string `json:"waits,omitempty"`
|
||||
}
|
||||
|
||||
// releaseOf is the release heard of module's version.
|
||||
func (r *releasedVersions) releaseOf(module, version string) (releaseHeard, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if h, ok := r.newest[module]; ok && h.Version == version {
|
||||
return h, true
|
||||
}
|
||||
for key, k := range r.seen {
|
||||
if strings.HasPrefix(key, module+"@") && k.Version == version {
|
||||
return releaseHeard{Version: version, Parts: k.Parts}, true
|
||||
}
|
||||
}
|
||||
return releaseHeard{}, false
|
||||
}
|
||||
|
||||
// deliveredAgain takes each ask of assigner_delivered_again, heard at once through the watch of the state delivered-again: it says
|
||||
// version-delivered again when every present machine placing the module runs that version healthy and removes the ask, else keeps
|
||||
// it with why it does not hold, which assigner_waits reads.
|
||||
func (p *passer) deliveredAgain(ctx context.Context) {
|
||||
if p.kvs[buckets.DeliveredAgain()] == nil {
|
||||
return
|
||||
}
|
||||
for key, value := range p.w.read(buckets.DeliveredAgain()) {
|
||||
var a againAsk
|
||||
if err := json.Unmarshal(value, &a); err != nil || a.Version == "" {
|
||||
p.logf("%s %s holds no ask, left: %v", buckets.DeliveredAgain(), key, err)
|
||||
continue
|
||||
}
|
||||
module := a.Module
|
||||
if module == "" {
|
||||
module = key
|
||||
}
|
||||
waits := ""
|
||||
rel, ok := p.w.released.releaseOf(module, a.Version)
|
||||
notRunning, away, placing := p.deliveryStanding(module, a.Version)
|
||||
switch {
|
||||
case !ok:
|
||||
waits = fmt.Sprintf("no version-released of %s %s is heard", module, a.Version)
|
||||
case placing == 0:
|
||||
waits = fmt.Sprintf("no machine places %s", module)
|
||||
case len(away) == placing:
|
||||
waits = fmt.Sprintf("every machine placing %s is away: %s", module, strings.Join(away, ", "))
|
||||
case len(notRunning) > 0:
|
||||
waits = fmt.Sprintf("%s %s does not run healthy on %s", module, a.Version, strings.Join(notRunning, ", "))
|
||||
}
|
||||
if waits == "" {
|
||||
if err := p.sayDelivered(module, rel, away); err != nil {
|
||||
p.logf("version-delivered of %s %s asked again is not said, and is asked at the next pass: %v", module, a.Version, err)
|
||||
continue
|
||||
}
|
||||
p.logf("version-delivered of %s %s said again, asked by %s: %s", module, a.Version, a.By, a.Why)
|
||||
p.remove(ctx, buckets.DeliveredAgain(), key)
|
||||
continue
|
||||
}
|
||||
if a.Waits != waits {
|
||||
a.Waits = waits
|
||||
p.put(ctx, buckets.DeliveredAgain(), key, func() ([]byte, error) { return json.Marshal(a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §376, §389, §498, A-189, A-246: a released version is said version-delivered once, when every present machine placing
|
||||
// it runs healthy the want naming it, a machine away named and not waited for, none when every placing machine is away; a dependent
|
||||
// whose want changes fingerprint waits by an edge of kind 2, 4 or 6 for its dependency's newest released version not yet said
|
||||
// delivered, said wait-held (step delivery, 60 minutes from the release) once and kept in delivery-waits, and wait-ended ran once
|
||||
// the version is delivered; a kind-1 edge, or a dependency with no version released, waits for nothing; one released by hand ends
|
||||
// released and its want is written.
|
||||
func TestADependentWaitsForItsDependencysDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
waits := &memKV{values: map[string][]byte{}}
|
||||
again := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.DeliveryWaits(): waits, buckets.DeliveredAgain(): again}, at)
|
||||
p.logf = t.Logf
|
||||
var events []said
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
events = append(events, said{event, id, raw})
|
||||
return nil
|
||||
}
|
||||
p.w.runningHeard = true
|
||||
placeOn(t, p.w, "ace", "db", "app")
|
||||
placeOn(t, p.w, "g14", "db")
|
||||
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
want := func(node, module, fp, version string) {
|
||||
key, _ := mwc.WantKey(node, module)
|
||||
raw, err := mwc.EncodeWant(mwc.Want{Fp: fp, Version: version, Self: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Wants(), key, raw)
|
||||
}
|
||||
runs := func(node, module, fp string) {
|
||||
raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "fingerprint": fp, "at": at})
|
||||
if err := p.w.hearRunning("running", raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
source := int64(3)
|
||||
release := func(module, version string) {
|
||||
source++
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{V: 1, Module: module, Version: version, Source: source, At: &at,
|
||||
Artifacts: []meshcatalog.VersionArtifact{{Artifact: "program", Digest: "sha256:d"}}})
|
||||
if err := p.w.released.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
saidOf := func(event string) []said {
|
||||
var out []said
|
||||
for _, e := range events {
|
||||
if e.event == event {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
want("ace", "app", fp("1"), "")
|
||||
|
||||
// No version of db is released: app's new fingerprint waits for nothing.
|
||||
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
|
||||
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("app waits with no version of db released")
|
||||
}
|
||||
release("db", "1.1.0")
|
||||
want("ace", "db", fp("d"), "1.1.0")
|
||||
want("g14", "db", fp("d"), "1.1.0")
|
||||
runs("ace", "db", fp("d"))
|
||||
// A kind-1 edge orders nothing.
|
||||
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 1, From: "db:program", To: "app:program"}})
|
||||
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("a kind-1 edge holds app")
|
||||
}
|
||||
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
|
||||
// app's want unchanged waits for nothing.
|
||||
if p.holdForDelivery(ctx, "app", fp("1"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("app's unchanged want waits")
|
||||
}
|
||||
// g14 places db and does not run 1.1.0 yet: not delivered, and app's new version waits, said once.
|
||||
p.judgeDelivered()
|
||||
if !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) || !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("app does not wait for db 1.1.0")
|
||||
}
|
||||
if len(saidOf(meshwants.EventVersionDelivered)) != 0 {
|
||||
t.Fatalf("db is said delivered while g14 does not run it: %+v", events)
|
||||
}
|
||||
held := saidOf(meshwants.EventWaitHeld)
|
||||
if len(held) != 1 {
|
||||
t.Fatalf("wait-held said %d times, want once", len(held))
|
||||
}
|
||||
var w meshwants.Wait
|
||||
_ = json.Unmarshal(held[0].body, &w)
|
||||
if w.Step != meshwants.WaitStepDelivery || w.Part != "db" || w.Module != "app" || w.Version != fp("2") || w.Bound != "1h0m0s" ||
|
||||
w.ClockFrom == nil || !w.ClockFrom.Equal(at) || len(w.Machines) != 1 || w.Machines[0] != "g14" || w.Words != "waiting for delivery of db 1.1.0 on g14" {
|
||||
t.Errorf("the wait said is %+v", w)
|
||||
}
|
||||
if _, ok := waits.values["app"]; !ok {
|
||||
t.Errorf("the wait is not kept in delivery-waits: %v", waits.values)
|
||||
}
|
||||
// g14 away: every present machine runs it, so it is delivered, g14 named away; once only.
|
||||
p.w.heardPresence("g14", true, at)
|
||||
p.judgeDelivered()
|
||||
p.judgeDelivered()
|
||||
delivered := saidOf(meshwants.EventVersionDelivered)
|
||||
var d meshwants.VersionDeliveredEvent
|
||||
if len(delivered) != 1 || json.Unmarshal(delivered[0].body, &d) != nil || d.Module != "db" || d.Version != "1.1.0" ||
|
||||
len(d.Away) != 1 || d.Away[0] != "g14" || d.Parts["db:program"] != "sha256:d" {
|
||||
t.Fatalf("version-delivered said %+v", delivered)
|
||||
}
|
||||
// The wait ends ran, and app's want goes on.
|
||||
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("app still waits once db 1.1.0 is delivered")
|
||||
}
|
||||
ended := saidOf(meshwants.EventWaitEnded)
|
||||
if len(ended) != 1 || json.Unmarshal(ended[0].body, &w) != nil || w.How != deliveryWaitRan {
|
||||
t.Errorf("wait-ended said %+v", ended)
|
||||
}
|
||||
if _, ok := waits.values["app"]; ok {
|
||||
t.Error("the ended wait is kept")
|
||||
}
|
||||
// A newer release with every placing machine away is not delivered, and its wait says the machines it waits to be back.
|
||||
release("db", "1.2.0")
|
||||
p.w.heardPresence("ace", true, at)
|
||||
p.judgeDelivered()
|
||||
if len(saidOf(meshwants.EventVersionDelivered)) != 1 {
|
||||
t.Error("a version is said delivered while every placing machine is away")
|
||||
}
|
||||
if !p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("app does not wait for db 1.2.0")
|
||||
}
|
||||
held = saidOf(meshwants.EventWaitHeld)
|
||||
if json.Unmarshal(held[len(held)-1].body, &w) != nil || w.Words != "waiting for ace, g14 to be back" {
|
||||
t.Errorf("the wait while every machine is away says %q", w.Words)
|
||||
}
|
||||
// Released by hand (assigner_release writes it): the wait ends released and app's want is written.
|
||||
var kept deliveryWaiting
|
||||
_ = json.Unmarshal(waits.values["app"], &kept)
|
||||
kept.Released = true
|
||||
raw, _ := json.Marshal(kept)
|
||||
p.w.put(buckets.DeliveryWaits(), "app", raw)
|
||||
if p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
|
||||
t.Fatal("a released wait holds app")
|
||||
}
|
||||
ended = saidOf(meshwants.EventWaitEnded)
|
||||
if json.Unmarshal(ended[len(ended)-1].body, &w) != nil || w.How != deliveryWaitReleased {
|
||||
t.Errorf("the released wait ended %+v", w)
|
||||
}
|
||||
// Presence: the later of machine-away and machine-back holds, whichever stream is read first.
|
||||
p.w.heardPresence("ace", false, at.Add(time.Minute))
|
||||
p.w.heardPresence("ace", true, at)
|
||||
if p.w.awayNow()["ace"] || !p.w.online()["ace"] {
|
||||
t.Error("an older machine-away read after a newer machine-back leaves ace away")
|
||||
}
|
||||
// A-250: an ask of assigner_delivered_again of 1.1.0, which ace (present) runs and g14 (away) does not, is said again and taken
|
||||
// away; one of 1.2.0, which no machine runs, is kept with why.
|
||||
ask := func(version string) {
|
||||
raw, _ := json.Marshal(againAsk{Module: "db", Version: version, By: "op", Why: "the catalog missed it"})
|
||||
p.w.put(buckets.DeliveredAgain(), "db", raw)
|
||||
}
|
||||
before := len(saidOf(meshwants.EventVersionDelivered))
|
||||
ask("1.1.0")
|
||||
p.deliveredAgain(ctx)
|
||||
if len(saidOf(meshwants.EventVersionDelivered)) != before+1 || p.w.has(buckets.DeliveredAgain(), "db") {
|
||||
t.Errorf("the ask of 1.1.0 said %d version-delivered and is kept %v", len(saidOf(meshwants.EventVersionDelivered))-before, p.w.has(buckets.DeliveredAgain(), "db"))
|
||||
}
|
||||
ask("1.2.0")
|
||||
p.deliveredAgain(ctx)
|
||||
var a againAsk
|
||||
if json.Unmarshal(p.w.value(buckets.DeliveredAgain(), "db"), &a) != nil || a.Waits != "db 1.2.0 does not run healthy on ace" ||
|
||||
len(saidOf(meshwants.EventVersionDelivered)) != before+1 {
|
||||
t.Errorf("the ask of 1.2.0 = %+v", a)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// engineWords is one node's key of the assigner's state engine-reads (design 47 H-A-104): the words its node-engine said it
|
||||
// reads in the last manifest-reads-changed the assigner heard of it, and when the engine said them.
|
||||
type engineWords struct {
|
||||
Words []string `json:"words"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// engineReadsKept is what the assigner holds of its state engine-reads (design 47 H-A-104, §772): written when it hears the
|
||||
// node-engine seat's manifest-reads-changed, and read back from the state's watch at the start, so a restart loses no node's
|
||||
// words; no verb is asked and no node's record is read in its place.
|
||||
type engineReadsKept struct {
|
||||
mu sync.Mutex
|
||||
by map[string]engineWords
|
||||
// put writes one node's key; nil keeps the words in memory only (the state could not be opened).
|
||||
put func(node string, value []byte) error
|
||||
// changed wakes a pass when a node's words changed; nil wakes nothing.
|
||||
changed func()
|
||||
}
|
||||
|
||||
// heard keeps what one manifest-reads-changed says of its node: written into the state first, then held.
|
||||
func (k *engineReadsKept) heard(e nodeengine.ManifestReadsChangedEvent) error {
|
||||
at, err := time.Parse(time.RFC3339Nano, e.At)
|
||||
if err != nil {
|
||||
return fmt.Errorf("at %q is no RFC 3339 time", e.At)
|
||||
}
|
||||
v := engineWords{Words: slices.Clone(e.ManifestReads), At: at}
|
||||
if k.put != nil {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.put(e.Node, raw); err != nil {
|
||||
return fmt.Errorf("the words of %s are not written into the state engine-reads: %w", e.Node, err)
|
||||
}
|
||||
}
|
||||
k.hold(e.Node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// take takes in one key of the state as its watch hands it: at the start every node's words, later the assigner's own writes
|
||||
// (which change nothing) and a key dropped.
|
||||
func (k *engineReadsKept) take(node string, raw []byte, deleted bool) error {
|
||||
if deleted {
|
||||
k.hold(node, engineWords{}, true)
|
||||
return nil
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return fmt.Errorf("the key %s of the state engine-reads does not read: %w", node, err)
|
||||
}
|
||||
k.hold(node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hold sets one node's words, waking a pass when they changed.
|
||||
func (k *engineReadsKept) hold(node string, v engineWords, deleted bool) {
|
||||
k.mu.Lock()
|
||||
old, had := k.by[node]
|
||||
switch {
|
||||
case deleted:
|
||||
delete(k.by, node)
|
||||
default:
|
||||
if k.by == nil {
|
||||
k.by = map[string]engineWords{}
|
||||
}
|
||||
k.by[node] = v
|
||||
}
|
||||
changed := (had && deleted) || (!deleted && (!had || !old.At.Equal(v.At) || !slices.Equal(old.Words, v.Words)))
|
||||
k.mu.Unlock()
|
||||
if changed && k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
}
|
||||
|
||||
// words is what node's engine said it reads, and whether the assigner holds its key; nil k holds none.
|
||||
func (k *engineReadsKept) words(node string) ([]string, bool) {
|
||||
if k == nil {
|
||||
return nil, false
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
v, ok := k.by[node]
|
||||
return slices.Clone(v.Words), ok
|
||||
}
|
||||
|
||||
// nodes is every node whose key the assigner holds, sorted.
|
||||
func (k *engineReadsKept) nodes() []string {
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(k.by))
|
||||
}
|
||||
|
||||
// engineReadsTool lists the assigner's state engine-reads: what each node's node-engine said it reads, and when.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
|
||||
// readEngineReads is the state engine-reads, every node's key or node's alone.
|
||||
func readEngineReads(st store, node string) (any, error) {
|
||||
keys := []string{node}
|
||||
if node == "" {
|
||||
var err error
|
||||
if keys, err = st.keys(engineReadsState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(keys)
|
||||
}
|
||||
out := map[string]engineWords{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := st.get(engineReadsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
if node != "" {
|
||||
return nil, fmt.Errorf("the assigner holds no words of %s: its node-engine said no %s it heard", node, nodeengine.EventManifestReadsChanged)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("the key %s does not read", key), err)
|
||||
}
|
||||
out[key] = v
|
||||
}
|
||||
return map[string]any{"nodes": out}, nil
|
||||
}
|
||||
|
||||
// engineReadsTools is the tool over the runtime's state.
|
||||
func engineReadsTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: engineReadsTool, Description: "What each node's node-engine says it reads of a manifest, as the " +
|
||||
"assigner keeps it in its state engine-reads (design 47 H-A-104): per node the words of the last manifest-reads-changed " +
|
||||
"it heard and when the engine said them. A node with no key states nothing the assigner heard, so its wants are held " +
|
||||
"awaiting its hand placement (§772). With node, that node's key alone.",
|
||||
Input: schema(map[string]string{"node": "one node; every node when left out"}, nil),
|
||||
Run: func(a map[string]any) (any, error) { return readEngineReads(st, arg(a, "node")) }}}
|
||||
}
|
||||
@@ -15,7 +15,21 @@ import (
|
||||
|
||||
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
|
||||
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
|
||||
f, is := g.objects.(assigngraph.Filler)
|
||||
return fillIn(ctx, g.objects)
|
||||
}
|
||||
|
||||
// passObjects is the store as the pass reads its fill: a handle of its own where one is given, so the pass's read never shares one
|
||||
// with a keeping that runs beside it (design 47 H-A-261).
|
||||
func (g *graphKeeper) passObjects() assigngraph.Objects {
|
||||
if g.reads != nil {
|
||||
return g.reads
|
||||
}
|
||||
return g.objects
|
||||
}
|
||||
|
||||
// fillIn is the fill of the bucket o, when it says one and has a cap.
|
||||
func fillIn(ctx context.Context, o assigngraph.Objects) (used, capacity int64, ok bool) {
|
||||
f, is := o.(assigngraph.Filler)
|
||||
if !is {
|
||||
return 0, 0, false
|
||||
}
|
||||
@@ -35,13 +49,13 @@ func (p *passer) sayFill(event, about string, used, capacity int64, refused stri
|
||||
// its cap), so a fill reached while the process was down is said too.
|
||||
func (p *passer) fillAtStart(ctx context.Context) {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case used >= capacity:
|
||||
g.full = true
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
|
||||
case used*5 >= capacity*4:
|
||||
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
|
||||
@@ -52,20 +66,20 @@ func (p *passer) fillAtStart(ctx context.Context) {
|
||||
// 80 % again. It answers true while the store is full.
|
||||
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if g.full && used*5 < capacity*4 {
|
||||
g.full = false
|
||||
if g.isFull() && used*5 < capacity*4 {
|
||||
g.setFull(false)
|
||||
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
|
||||
g.full = false
|
||||
if used+assigngraph.NewBytes(ctx, g.passObjects(), c) <= capacity {
|
||||
g.setFull(false)
|
||||
return false
|
||||
}
|
||||
if !g.full {
|
||||
g.full = true
|
||||
if !g.isFull() {
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
|
||||
|
||||
@@ -61,14 +61,14 @@ func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
o.base = 700
|
||||
o.base, o.cap = 8500, 10000 // past 80 %, with room for the next revision, which the pass judges itself (H-A-261)
|
||||
p.graph.started = false
|
||||
p.keepGraph(ctx)
|
||||
p.keepGraph(ctx) // a start again: its first pass judges the fill
|
||||
if len(events) != 1 || events[0] != meshwants.EventFilling {
|
||||
t.Fatalf("at the start past 80 %%: %v", events)
|
||||
}
|
||||
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
|
||||
o.base = 990
|
||||
o.base, o.cap = 990, 1000
|
||||
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
|
||||
t.Fatalf("a revision that does not fit: events %v", events)
|
||||
}
|
||||
@@ -76,8 +76,15 @@ func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
if n := len(events); n != 2 {
|
||||
t.Errorf("full is said once: %v", events)
|
||||
}
|
||||
// H-A-261: the verdict is the pass's own: with the store full the pass writes no want, though a revision is kept.
|
||||
if held := p.keepGraph(ctx); held != heldFull {
|
||||
t.Errorf("a pass with the store full answers %q, want %q", held, heldFull)
|
||||
}
|
||||
o.base = 100
|
||||
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
|
||||
t.Errorf("room: events %v", events)
|
||||
}
|
||||
if held := p.keepGraph(ctx); held != "" {
|
||||
t.Errorf("a pass with room and a revision kept answers %q, want the wants written", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,18 +5,25 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
|
||||
@@ -25,83 +32,288 @@ import (
|
||||
// that follows are written from the revision it holds.
|
||||
type graphKeeper struct {
|
||||
objects assigngraph.Objects
|
||||
// reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects.
|
||||
reads assigngraph.Objects
|
||||
// halves is every followed branch's catalog graph revision held now.
|
||||
halves func() map[string]mcc.Half
|
||||
// last is the revision last computed, so a pass whose inputs did not change keeps and says nothing.
|
||||
last meshwants.AssignmentRevision
|
||||
has bool
|
||||
halves func() map[string]mcatc.Half
|
||||
// mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259).
|
||||
mu sync.Mutex
|
||||
// handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false
|
||||
// until one is, and again when its keeping failed or found the store full, so the next pass hands it again.
|
||||
handedRev meshwants.AssignmentRevision
|
||||
handed bool
|
||||
// handedOnce says a revision was handed: the first carries the start's removal (H-A-261).
|
||||
handedOnce bool
|
||||
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
|
||||
kept meshwants.AssignmentRevision
|
||||
source int64
|
||||
ok bool
|
||||
said string
|
||||
// cycled is the cycles last said (assigngraph.Cycles).
|
||||
cycled string
|
||||
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
|
||||
full, started bool
|
||||
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
|
||||
pins *pinKeeper
|
||||
// computed is the revision computed by the last pass, with its entries: the edges the wants wait on (actingwaits.go).
|
||||
computed assigngraph.Computed
|
||||
// revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision.
|
||||
revisions *revisionRemover
|
||||
// computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one.
|
||||
computed assigngraph.Computed
|
||||
hasComputed bool
|
||||
// keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow).
|
||||
keepMu sync.Mutex
|
||||
// The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at
|
||||
// once, in the caller), and wake makes a pass once a keeping changed what the pass reads.
|
||||
need *keepNeed
|
||||
keeping bool
|
||||
background func(func())
|
||||
wake func()
|
||||
// A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures
|
||||
// counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)).
|
||||
retry time.Duration
|
||||
after func(d time.Duration, f func())
|
||||
failures int
|
||||
}
|
||||
|
||||
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
|
||||
func (p *passer) keepGraph(ctx context.Context) {
|
||||
// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline).
|
||||
const (
|
||||
keepingRetrySetting = "keeping-retry"
|
||||
defaultKeepingRetry = 30 * time.Second
|
||||
kindKeepingFailed = "keeping-failed"
|
||||
// keepingFailedAt is the failures in a row that raise keeping-failed.
|
||||
keepingFailedAt = 3
|
||||
)
|
||||
|
||||
// keepingRetry reads the setting keeping-retry (§781).
|
||||
func keepingRetry() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged.
|
||||
type keepNeed struct {
|
||||
c *assigngraph.Computed
|
||||
judge bool
|
||||
}
|
||||
|
||||
// keptNow is the revision the wants follow, with its trunk source, and whether one is kept.
|
||||
func (g *graphKeeper) keptNow() (meshwants.AssignmentRevision, int64, bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.kept, g.source, g.ok
|
||||
}
|
||||
|
||||
// isFull says the store did not hold the last revision handed over, so no want is written (design 47 G §181).
|
||||
func (g *graphKeeper) isFull() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.full
|
||||
}
|
||||
|
||||
func (g *graphKeeper) setFull(full bool) {
|
||||
g.mu.Lock()
|
||||
g.full = full
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
// heldNoRevision is the pass's words while no assignment graph revision is kept: it writes no want (design 47 H-A-261).
|
||||
const heldNoRevision = "held: no assignment graph revision is kept yet, so no want is written until the first keeping ends (H-A-261)"
|
||||
|
||||
// heldFull is the pass's words while the next revision does not fit its store (design 47 G §181).
|
||||
const heldFull = "the assignment graph store is full: no want is written (design 47 G §181)"
|
||||
|
||||
// keepGraph computes the revision of this pass, judges the store's fill itself, and hands the revision to the keeping when the
|
||||
// pass's inputs differ from the last handed. The keeping and the removing of revisions run off the pass loop, one at a time, and the
|
||||
// pass never waits on them: it judges and writes the wants from the graph it holds (design 47 H-A-259). It answers why the pass
|
||||
// writes no want, or "" when it writes them: none while the next revision does not fit the store (G §181), and none while no
|
||||
// revision is kept (H-A-261), whose keeping wakes a pass when it ends.
|
||||
func (p *passer) keepGraph(ctx context.Context) string {
|
||||
g := p.graph
|
||||
if g == nil || g.objects == nil || g.halves == nil {
|
||||
return
|
||||
return ""
|
||||
}
|
||||
c := assigngraph.Compute(p.graphInputs())
|
||||
g.computed = c
|
||||
if p.ends != nil {
|
||||
p.ends.setRunner(assigngraph.Runner(slices.Collect(maps.Values(c.Entries)))) // the runner's member of a group goes first (A-334)
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.computed, g.hasComputed = c, true
|
||||
g.mu.Unlock()
|
||||
for _, u := range c.Uncomputed {
|
||||
if g.said != u {
|
||||
g.said = u
|
||||
p.logf("the assignment graph leaves out %s", u)
|
||||
}
|
||||
}
|
||||
if cycles := strings.Join(c.Cycles, "; "); cycles != g.cycled {
|
||||
g.cycled = cycles
|
||||
if cycles != "" {
|
||||
p.logf("the assignment graph holds a cycle among the acting waits, within one step or across steps: %s", cycles)
|
||||
}
|
||||
}
|
||||
if !g.started {
|
||||
if !g.started { // the first pass judges the fill once (graphfill.go)
|
||||
g.started = true
|
||||
p.fillAtStart(ctx)
|
||||
}
|
||||
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
|
||||
return
|
||||
}
|
||||
used, _, _ := g.fillOf(ctx)
|
||||
if p.fullBefore(ctx, c) {
|
||||
return
|
||||
g.mu.Lock()
|
||||
g.handed = false // handed once it fits
|
||||
g.mu.Unlock()
|
||||
return heldFull
|
||||
}
|
||||
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
|
||||
if err != nil {
|
||||
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
|
||||
return
|
||||
g.mu.Lock()
|
||||
same := g.handed && assigngraph.Same(g.handedRev, c.Revision)
|
||||
first := !g.handedOnce
|
||||
if !same {
|
||||
g.handedRev, g.handed, g.handedOnce = c.Revision, true, true
|
||||
}
|
||||
g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true
|
||||
g.source = 0
|
||||
for _, cat := range rev.Catalogs {
|
||||
if cat.Branch == p.w.trunkName() {
|
||||
g.source = cat.Source
|
||||
g.mu.Unlock()
|
||||
if !same {
|
||||
// The start's removal of the revisions not current folds into the first keeping: it keeps the first pass's revision, then
|
||||
// removes (H-A-261).
|
||||
p.needKeeping(ctx, keepNeed{c: &c, judge: first})
|
||||
}
|
||||
if _, _, ok := g.keptNow(); !ok {
|
||||
return heldNoRevision
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// needKeeping hands a need to the keeping: one keeping runs at a time, and a need that comes while one runs is folded into the one
|
||||
// that waits (the newest revision, and a judging asked by either), so it runs once more (design 47 H-A-259).
|
||||
func (p *passer) needKeeping(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
if w := g.need; w != nil {
|
||||
n.judge = n.judge || w.judge
|
||||
if n.c == nil {
|
||||
n.c = w.c
|
||||
}
|
||||
}
|
||||
if !fresh {
|
||||
g.need = &n
|
||||
if g.keeping {
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.fillingAfter(ctx, used)
|
||||
if g.pins != nil {
|
||||
g.pins.judge() // the revision before ceased to be current
|
||||
g.keeping = true
|
||||
g.mu.Unlock()
|
||||
run := func() {
|
||||
for {
|
||||
g.mu.Lock()
|
||||
next := g.need
|
||||
g.need = nil
|
||||
if next == nil {
|
||||
g.keeping = false
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.mu.Unlock()
|
||||
p.keepOne(ctx, *next)
|
||||
}
|
||||
}
|
||||
if g.background != nil {
|
||||
g.background(run)
|
||||
return
|
||||
}
|
||||
run()
|
||||
}
|
||||
|
||||
// keepOne keeps one computed revision, says assignment-revision for a revision kept now and removes the
|
||||
// revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads.
|
||||
func (p *passer) keepOne(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
g.keepMu.Lock()
|
||||
defer g.keepMu.Unlock()
|
||||
start := time.Now()
|
||||
changed := false
|
||||
defer func() {
|
||||
p.logf("the keeping of the assignment graph took %s", time.Since(start).Round(time.Millisecond))
|
||||
if changed && g.background != nil && g.wake != nil {
|
||||
g.wake()
|
||||
}
|
||||
}()
|
||||
if n.c != nil {
|
||||
_, _, had := g.keptNow()
|
||||
fresh := p.keepRevision(ctx, *n.c)
|
||||
_, _, has := g.keptNow()
|
||||
n.judge = n.judge || fresh
|
||||
changed = fresh || had != has // the first revision kept: the pass writes the wants it held (H-A-261)
|
||||
}
|
||||
if n.judge && g.revisions != nil {
|
||||
g.revisions.judge() // the revision before ceased to be current
|
||||
}
|
||||
}
|
||||
|
||||
// keepRevision keeps c when the store holds it, and says whether a revision was kept now.
|
||||
func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool {
|
||||
fresh, _ := p.keepRevisionOf(ctx, c)
|
||||
return fresh
|
||||
}
|
||||
|
||||
// keepRevisionOf is keepRevision answering the fault the keeping met too.
|
||||
func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) {
|
||||
g := p.graph
|
||||
again := func() { // the next pass hands this revision again
|
||||
g.mu.Lock()
|
||||
g.handed = false
|
||||
g.mu.Unlock()
|
||||
}
|
||||
used, _, _ := g.fillOf(ctx)
|
||||
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
|
||||
if err != nil {
|
||||
again()
|
||||
p.keepingFailed(ctx, c, err)
|
||||
return false, err
|
||||
}
|
||||
p.keepingSucceeded()
|
||||
var source int64
|
||||
for _, cat := range rev.Catalogs {
|
||||
if cat.Branch == p.w.trunkName() {
|
||||
source = cat.Source
|
||||
}
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.kept, g.ok, g.source = rev, true, source
|
||||
g.mu.Unlock()
|
||||
if !fresh {
|
||||
return false, nil
|
||||
}
|
||||
p.fillingAfter(ctx, used)
|
||||
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
|
||||
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
|
||||
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed
|
||||
// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed
|
||||
// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls.
|
||||
func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) {
|
||||
g := p.graph
|
||||
if strings.TrimSpace(in.Why) == "" {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now")
|
||||
}
|
||||
if g == nil || g.objects == nil {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here")
|
||||
}
|
||||
g.mu.Lock()
|
||||
c, ok := g.computed, g.hasComputed
|
||||
g.mu.Unlock()
|
||||
if !ok {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet")
|
||||
}
|
||||
g.keepMu.Lock()
|
||||
defer g.keepMu.Unlock()
|
||||
p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why)
|
||||
if _, err := p.keepRevisionOf(ctx, c); err != nil {
|
||||
return meshwants.KeepNowAnswer{Fault: err.Error()}, nil
|
||||
}
|
||||
rev, _, _ := g.keptNow()
|
||||
return meshwants.KeepNowAnswer{Revision: &rev}, nil
|
||||
}
|
||||
|
||||
// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now.
|
||||
func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.KeepNowInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return p.keepNow(ctx, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
|
||||
@@ -131,16 +343,12 @@ func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesI
|
||||
slices.Sort(known)
|
||||
var out []meshwants.AssignmentRevisionCapabilitiesItem
|
||||
for _, node := range known {
|
||||
read := p.reads != nil && p.reads.Known(node)
|
||||
reads, read := p.reads.words(node)
|
||||
rec, recorded := records[node]
|
||||
if !read && !recorded {
|
||||
continue
|
||||
}
|
||||
var reads []string
|
||||
if read {
|
||||
reads = slices.Clone(p.reads.Reads([]string{node})[node])
|
||||
slices.Sort(reads)
|
||||
}
|
||||
slices.Sort(reads)
|
||||
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
|
||||
Fingerprint: rec.RunnerFingerprint}
|
||||
slices.Sort(item.Readings)
|
||||
@@ -158,19 +366,25 @@ func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesI
|
||||
|
||||
// sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is
|
||||
// said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there.
|
||||
func (p *passer) sayWanted(node, module, fp string, removed bool) {
|
||||
func (p *passer) sayWanted(node, module, fp, pipeline string, removed bool) {
|
||||
if p.publishWants == nil {
|
||||
return
|
||||
}
|
||||
g := p.graph
|
||||
if g == nil || !g.ok {
|
||||
var kept meshwants.AssignmentRevision
|
||||
var source int64
|
||||
ok := false
|
||||
if g != nil {
|
||||
kept, source, ok = g.keptNow()
|
||||
}
|
||||
if !ok {
|
||||
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at}
|
||||
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: kept.Revision, Source: source, At: &at}
|
||||
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
|
||||
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, g.kept.Revision, at.Format(time.RFC3339Nano))
|
||||
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, kept.Revision, at.Format(time.RFC3339Nano))
|
||||
if err := p.wantsSay(event, body, id); err != nil {
|
||||
p.logf("wanted of %s on %s is not said: %v", module, node, err)
|
||||
}
|
||||
@@ -178,15 +392,121 @@ func (p *passer) sayWanted(node, module, fp string, removed bool) {
|
||||
|
||||
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
|
||||
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
|
||||
// version and the parts of the `version-released` whose source is the one the kept assignment graph revision was computed from (A-67).
|
||||
func (p *passer) namedBuild(module, trunkFP string) func(fp string) (assign.KeptBuild, bool) {
|
||||
// version and the parts of the `version-released` whose parts are the digests its build.ended made (A-421): the catalog stamps no
|
||||
// source on an entry (design 47 H §764, H-A-48). A trunk entry whose release is not heard is said, once, naming which was tried.
|
||||
func (p *passer) namedBuild(module, trunkFP string, unreleased ...bool) func(fp string) (assign.KeptBuild, bool) {
|
||||
return func(fp string) (assign.KeptBuild, bool) {
|
||||
if k, ok := p.w.ends.partsOf(module, fp); ok {
|
||||
return k, true
|
||||
}
|
||||
if fp != trunkFP || p.graph == nil || !p.graph.ok {
|
||||
if len(unreleased) > 0 && unreleased[0] && fp == trunkFP {
|
||||
// Wanted with no passed pipeline (the holder of mesh-catalog, the runner's first start): no pipeline released it,
|
||||
// so its want names the parts its build.ended made, until a release of it is heard (design 47 G §16, A-428).
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
if made := p.w.ends.madeOf(module, fp); len(made) > 0 {
|
||||
return assign.KeptBuild{Parts: made}, true
|
||||
}
|
||||
}
|
||||
if fp != trunkFP {
|
||||
// A want held back at an older build (rule 2: no pipeline passed at the trunk's newer fingerprint) names that build's
|
||||
// release when one names exactly the digests it made (novox 2026-11-16: mesh-hand-acts 0.1.13 held at 4eb3cf48 after
|
||||
// the mesh-sdk merge moved its fingerprint).
|
||||
return p.w.released.withParts(module, p.w.ends.madeOf(module, fp))
|
||||
}
|
||||
if _, _, kept := p.graphKept(); !kept {
|
||||
p.unheardRelease(module, fp, "no assignment graph revision is kept yet, so no release is looked up")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
return p.w.released.of(module, p.graph.source)
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
p.unheardRelease(module, fp, "no version-released of it names the parts its build.ended made")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// unheardRelease says once per module, fingerprint and why that a trunk entry's release is not heard, so its wants and candidates
|
||||
// carry no version and no parts (design 47 G A-419).
|
||||
func (p *passer) unheardRelease(module, fp, why string) {
|
||||
key := "release " + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the release of %s at fingerprint %s is not heard: %s; its wants and candidates carry no version and no parts", module, fp, why)
|
||||
}
|
||||
}
|
||||
|
||||
// graphKept is the revision the wants follow, with its trunk source, and whether one is kept; with no graph kept, none is.
|
||||
func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) {
|
||||
if p.graph == nil {
|
||||
return meshwants.AssignmentRevision{}, 0, false
|
||||
}
|
||||
return p.graph.keptNow()
|
||||
}
|
||||
|
||||
// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer
|
||||
// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module).
|
||||
func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
g.failures++
|
||||
n, retry, after := g.failures, g.retry, g.after
|
||||
g.mu.Unlock()
|
||||
if retry <= 0 {
|
||||
retry = defaultKeepingRetry
|
||||
}
|
||||
if after == nil {
|
||||
after = func(d time.Duration, f func()) { time.AfterFunc(d, f) }
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err)
|
||||
if n == keepingFailedAt && p.findings.Put != nil {
|
||||
if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", kindKeepingFailed, err)
|
||||
}
|
||||
}
|
||||
after(retry, func() {
|
||||
g.mu.Lock()
|
||||
newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision)
|
||||
if !newer {
|
||||
g.handedRev, g.handed = c.Revision, true
|
||||
}
|
||||
g.mu.Unlock()
|
||||
if !newer {
|
||||
p.needKeeping(ctx, keepNeed{c: &c})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared.
|
||||
func (p *passer) keepingSucceeded() {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
n := g.failures
|
||||
g.failures = 0
|
||||
g.mu.Unlock()
|
||||
if n >= keepingFailedAt && p.findings.Delete != nil {
|
||||
o := keepingFailedFinding(n, nil)
|
||||
if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil {
|
||||
p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)).
|
||||
func keepingFailedFinding(n int, err error) assign.Observed {
|
||||
summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n)
|
||||
if err != nil {
|
||||
summary += ": " + err.Error()
|
||||
}
|
||||
return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed,
|
||||
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept",
|
||||
Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.",
|
||||
Source: "assigner." + kindKeepingFailed}
|
||||
}
|
||||
|
||||
@@ -3,17 +3,25 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
@@ -88,6 +96,35 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
|
||||
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
|
||||
}
|
||||
// Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a
|
||||
// want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands.
|
||||
t.Run("a want's run", func(t *testing.T) {
|
||||
wkey, _ := mwc.WantKey("g14", "db")
|
||||
stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w }
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Pipeline != "main.12" {
|
||||
t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w)
|
||||
}
|
||||
puts := kv.puts
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts {
|
||||
t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts)
|
||||
}
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
if kv.puts != puts {
|
||||
t.Error("a want that stands was written again")
|
||||
}
|
||||
skey, _ := mwc.WantKey("shanks", "db")
|
||||
p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"})
|
||||
if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" {
|
||||
t.Errorf("a put's want is %+v, want the newest source held, 9", w)
|
||||
}
|
||||
// a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.0" {
|
||||
t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w)
|
||||
}
|
||||
})
|
||||
|
||||
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
|
||||
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
|
||||
@@ -132,3 +169,227 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
|
||||
// A listing that meets the gate closes entered: the keeping took its need and runs it (H-A-476).
|
||||
type blockedObjects struct {
|
||||
memObjects
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
gate, entered := b.gate, b.entered
|
||||
if gate != nil && entered != nil {
|
||||
close(entered)
|
||||
b.entered = nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
// block stands a gate, opened by the function it answers.
|
||||
func (b *blockedObjects) block() func() {
|
||||
gate := make(chan struct{})
|
||||
b.mu.Lock()
|
||||
b.gate, b.entered = gate, make(chan struct{})
|
||||
b.mu.Unlock()
|
||||
return func() {
|
||||
b.mu.Lock()
|
||||
b.gate, b.entered = nil, nil
|
||||
b.mu.Unlock()
|
||||
close(gate)
|
||||
}
|
||||
}
|
||||
|
||||
// awaitBlocked returns once a keeping meets the standing gate, so what the keeping took is read after it took it, and fails the
|
||||
// test when none did within bound (H-A-476: a test waits on the signal of the work it reads after, never on an order it assumes).
|
||||
func (b *blockedObjects) awaitBlocked(t *testing.T, bound time.Duration, what string) {
|
||||
t.Helper()
|
||||
b.mu.Lock()
|
||||
entered := b.entered
|
||||
b.mu.Unlock()
|
||||
if entered == nil {
|
||||
return // met already
|
||||
}
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(bound):
|
||||
t.Fatalf("no keeping met the gate within %s: %s", bound, what)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
|
||||
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
|
||||
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
|
||||
// more. Each pass and each keeping says its duration.
|
||||
func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
bound := testbus.SetupBound(t) // every wait on a goroutine's work, never a value in the test (H-A-476)
|
||||
var mu sync.Mutex
|
||||
var journal strings.Builder
|
||||
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
|
||||
said := func(words string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return strings.Contains(journal.String(), words)
|
||||
}
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := mcc.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: mcc.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 1, mpr.StepResultOutcomePassed)))
|
||||
objects := &blockedObjects{memObjects: memObjects{}}
|
||||
var kept sync.WaitGroup
|
||||
woke := make(chan struct{}, 8)
|
||||
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
|
||||
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} },
|
||||
background: func(f func()) { kept.Go(f) }, wake: func() { woke <- struct{}{} }}
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
done := make(chan struct{})
|
||||
go func() { p.pass(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the pass waits on the keeping of the assignment graph")
|
||||
}
|
||||
}
|
||||
wantOf := func(node string) (mwc.Want, error) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
return mwc.DecodeWant(wantsKV.values[key])
|
||||
}
|
||||
open := objects.block()
|
||||
t.Cleanup(func() { // the keeping ends before the test does, and says its duration
|
||||
objects.mu.Lock()
|
||||
standing := objects.gate != nil
|
||||
objects.mu.Unlock()
|
||||
if standing {
|
||||
open()
|
||||
}
|
||||
kept.Wait()
|
||||
if !said("the keeping of the assignment graph took ") {
|
||||
t.Error("the keeping does not say its duration")
|
||||
}
|
||||
})
|
||||
|
||||
pass() // the start: the first keeping is blocked
|
||||
if _, err := wantOf("ace"); err == nil {
|
||||
t.Error("a pass at the start wrote a want before an assignment graph revision is kept")
|
||||
}
|
||||
open()
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the first keeping's end wakes no pass")
|
||||
}
|
||||
pass()
|
||||
if w, err := wantOf("ace"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on ace = %+v (%v), want the passed version once a revision is kept", w, err)
|
||||
}
|
||||
|
||||
open = objects.block() // a keeping of new inputs is blocked
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pass()
|
||||
pass() // the same inputs again
|
||||
objects.awaitBlocked(t, bound, "the keeping of new inputs never took its need") // H-A-476: read the need once it was taken
|
||||
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
|
||||
}
|
||||
p.graph.mu.Lock()
|
||||
keeping, waiting := p.graph.keeping, p.graph.need
|
||||
p.graph.mu.Unlock()
|
||||
if !keeping || waiting != nil {
|
||||
t.Errorf("keeping %v, need waiting %+v: the pass of the same inputs handed its revision again", keeping, waiting)
|
||||
}
|
||||
if !said("the pass took ") {
|
||||
t.Error("the pass does not say its duration")
|
||||
}
|
||||
open()
|
||||
kept.Wait()
|
||||
// Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third
|
||||
// failure in a row, cleared at the next that succeeds.
|
||||
t.Run("a failed keeping runs again", func(t *testing.T) {
|
||||
failing := &failingObjects{blockedObjects: objects, fail: 3}
|
||||
var retries []time.Duration
|
||||
var put []mcseat.Finding
|
||||
var deleted []string
|
||||
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil },
|
||||
Delete: func(key string) error { deleted = append(deleted, key); return nil }}
|
||||
p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second
|
||||
p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() }
|
||||
placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed
|
||||
pass()
|
||||
if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" ||
|
||||
len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 {
|
||||
t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures)
|
||||
}
|
||||
if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 {
|
||||
t.Fatal("the revision is not kept once the store takes it")
|
||||
}
|
||||
// keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why.
|
||||
if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil {
|
||||
t.Error("a keeping by hand with no why runs")
|
||||
}
|
||||
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" {
|
||||
t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err)
|
||||
}
|
||||
failing.fail, retries = 2, nil
|
||||
p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) }
|
||||
placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand
|
||||
pass()
|
||||
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 {
|
||||
t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries)
|
||||
}
|
||||
var asked []string
|
||||
tool := keepNowTools(func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
})[0]
|
||||
if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") ||
|
||||
asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` {
|
||||
t.Errorf("assigner_keep_now: %v, asked %v", err, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// failingObjects refuses the next fail puts, then is objects.
|
||||
type failingObjects struct {
|
||||
*blockedObjects
|
||||
fail int
|
||||
}
|
||||
|
||||
func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
if f.fail > 0 {
|
||||
f.fail--
|
||||
return errors.New("the store answers nothing")
|
||||
}
|
||||
return f.blockedObjects.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
@@ -35,6 +37,10 @@ type halves struct {
|
||||
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
|
||||
settled chan<- struct{}
|
||||
told bool
|
||||
|
||||
// regMu orders the registrations told to registered: the ones held when it is set, then each change (H-A-270 (3)).
|
||||
regMu sync.Mutex
|
||||
registered func(reg meshcatalog.Registration, gone bool)
|
||||
}
|
||||
|
||||
// follow makes the branch follower on the reader; the reader's start asks the first views.
|
||||
@@ -48,12 +54,51 @@ func (h *halves) follow() error {
|
||||
h.changed(branch, module, entry, deleted)
|
||||
}
|
||||
b.Adopted = h.adopted
|
||||
// The one handler of registration-changed is the follower's (H-A-270 (3)): the pipeline ends are told each change from it,
|
||||
// never from a second handler on the hub.
|
||||
reconcile := b.Registrations.Follower.Changed
|
||||
b.Registrations.Follower.Changed = func(module string, value json.RawMessage, gone bool, at time.Time) {
|
||||
if reconcile != nil {
|
||||
reconcile(module, value, gone, at)
|
||||
}
|
||||
h.tellRegistration(module, value, gone)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.branches = b
|
||||
h.mu.Unlock()
|
||||
h.settleWhenNoBranch()
|
||||
return nil
|
||||
}
|
||||
|
||||
// settleWhenNoBranch tells the first pass once, when the registrations and the gate's source are answered and name no branch: the
|
||||
// trunk is none-yet and the world is empty, as the catalog client's Ready says it (design 47 G, A-372). A mesh that names a branch is
|
||||
// told by adopted, when that branch's first view is answered.
|
||||
func (h *halves) settleWhenNoBranch() {
|
||||
if h.settled == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
go func() {
|
||||
select {
|
||||
case <-b.FirstViews(h.ctx):
|
||||
case <-h.ctx.Done():
|
||||
return
|
||||
}
|
||||
if len(b.Followed()) > 0 || b.Ready() != nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
told := h.told
|
||||
h.told = true
|
||||
h.mu.Unlock()
|
||||
if !told {
|
||||
h.settled <- struct{}{}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// first is the first watched branch followed, the trunk's: "" while none is.
|
||||
func (h *halves) first() string {
|
||||
h.mu.Lock()
|
||||
@@ -161,3 +206,61 @@ func (h *halves) views() map[string]mcc.Half {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the
|
||||
// order `watch` added it). Once registrations are held, a module none names has none: it is unlisted (H-A-144). Only before the
|
||||
// first registrations are held do the gate's source's branches stand in (the branches followed then), in the order the source
|
||||
// gives them, by their names escaped as its keys are (design 47 G §532, H-A-143, H-A-144).
|
||||
func (h *halves) watchedOf(module string) []string {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if b.Registrations != nil && len(b.Registrations.All()) > 0 {
|
||||
if reg, ok := b.Registrations.Registration(module); ok {
|
||||
return slices.Clone(reg.Branches)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := b.Followed()
|
||||
slices.SortStableFunc(out, func(x, y string) int { return strings.Compare(keys.EscapeName(x), keys.EscapeName(y)) })
|
||||
return out
|
||||
}
|
||||
|
||||
// tellRegistration tells registered of one registration the follower holds now, or of one removed.
|
||||
func (h *halves) tellRegistration(module string, value json.RawMessage, gone bool) {
|
||||
h.regMu.Lock()
|
||||
defer h.regMu.Unlock()
|
||||
if h.registered == nil {
|
||||
return
|
||||
}
|
||||
reg := meshcatalog.Registration{Module: module}
|
||||
if !gone {
|
||||
if err := json.Unmarshal(value, ®); err != nil {
|
||||
h.logf("the registration of %s does not read: %v", module, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
h.registered(reg, gone)
|
||||
}
|
||||
|
||||
// followRegistrations tells registered of every registration the branch follower holds and then of each change it takes
|
||||
// (design 47 G §353): the registrations are followed once, by the follower's one handler of registration-changed (H-A-270
|
||||
// (3)). It says false when no follower is made.
|
||||
func (h *halves) followRegistrations(registered func(meshcatalog.Registration, bool)) bool {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil || b.Registrations == nil {
|
||||
return false
|
||||
}
|
||||
h.regMu.Lock()
|
||||
defer h.regMu.Unlock()
|
||||
h.registered = registered
|
||||
for _, reg := range b.Registrations.All() {
|
||||
registered(reg, false)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// Design 47 G A-372: the assigner's wait for its catalog half asks the catalog client's Ready as well: a mesh whose registrations and
|
||||
// gate's source are answered and name no branch is none-yet, and the first pass is told so once; before they answer it is not told.
|
||||
func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
if key == mcc.RegistrationsVerb {
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: []seat.Registration{}})
|
||||
}
|
||||
return json.Marshal(map[string][]string{"branches": {}})
|
||||
}
|
||||
for _, answered := range []bool{false, true} {
|
||||
settled := make(chan struct{}, 2)
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx, settled: settled}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if answered {
|
||||
for _, f := range reader.Followers() {
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("a mesh that names no branch never settled the first pass")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
t.Errorf("answered %v: the first pass was told twice or before the first views", answered)
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, H-A-144, G §532: a module's watched branches are its registration's, in its order; once registrations are held a
|
||||
// module none names has none; before the first registrations the gate's source's branches stand in, in the source's order.
|
||||
// H-A-270 (3): the pipeline ends are told the registrations by the same follower, whether set before its first view or after.
|
||||
func TestAModulesWatchedBranchesAreItsRegistrations(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
regs []seat.Registration
|
||||
module string
|
||||
want []string
|
||||
}{
|
||||
{"its registration's order", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "link2pay", []string{"nox-mesh", "main"}},
|
||||
{"no registration names it", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "checker", nil},
|
||||
{"before the first registrations, the source's", []seat.Registration{}, "checker", []string{"main", "master"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case mcc.RegistrationsVerb:
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: tc.regs})
|
||||
case mcc.SourceVerb:
|
||||
return json.Marshal(map[string][]string{"branches": {"master", "main"}})
|
||||
}
|
||||
return nil, errors.New("not answered here")
|
||||
}
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var live, held []string
|
||||
if !h.followRegistrations(func(reg seat.Registration, gone bool) { live = append(live, reg.Module) }) {
|
||||
t.Fatal("no follower of the registrations")
|
||||
}
|
||||
for _, f := range reader.Followers()[:2] { // the registrations and the gate's source
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.followRegistrations(func(reg seat.Registration, gone bool) { held = append(held, reg.Module) })
|
||||
var regs []string
|
||||
for _, r := range tc.regs {
|
||||
regs = append(regs, r.Module)
|
||||
}
|
||||
if !slices.Equal(live, regs) || !slices.Equal(held, regs) {
|
||||
t.Errorf("the ends were told %v at the first view and %v held, want %v", live, held, regs)
|
||||
}
|
||||
if got := h.watchedOf(tc.module); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("the watched branches of %s are %v, want %v", tc.module, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -24,14 +25,17 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
|
||||
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
|
||||
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
|
||||
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
|
||||
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
|
||||
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
|
||||
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
|
||||
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
|
||||
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
|
||||
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
|
||||
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
|
||||
|
||||
// inputsTool is the tool that reads the kept inputs.
|
||||
@@ -44,10 +48,39 @@ const (
|
||||
inputsSwept = 20
|
||||
inputsTime = "20060102T150405Z"
|
||||
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
|
||||
// holding the input's own key, so the build seat's step-ended of that ask finds it.
|
||||
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
|
||||
inputsAskPrefix = "ask."
|
||||
)
|
||||
|
||||
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
|
||||
type inputAsk struct {
|
||||
Digest string `json:"digest"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
|
||||
// carries its time in the key itself.
|
||||
func readAsk(raw []byte) (digest string, at time.Time) {
|
||||
var a inputAsk
|
||||
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
|
||||
at, _ = time.Parse(inputsTime, a.At)
|
||||
return a.Digest, at
|
||||
}
|
||||
digest = string(raw)
|
||||
t, _, _ := strings.Cut(digest, ".")
|
||||
at, _ = time.Parse(inputsTime, t)
|
||||
return digest, at
|
||||
}
|
||||
|
||||
// isDigest is whether a key is an input's own key: 64 hex digits.
|
||||
func isDigest(key string) bool {
|
||||
if len(key) != 64 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// digestOf is an input's digest as the ask names it: its sha256 in hex.
|
||||
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
|
||||
|
||||
@@ -63,8 +96,9 @@ type inputKeeper struct {
|
||||
used func() (int64, error)
|
||||
}
|
||||
|
||||
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
|
||||
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
|
||||
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
|
||||
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
|
||||
// first, so a sweep never finds the input unnamed.
|
||||
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
|
||||
if ask == "" {
|
||||
return "", errors.New("name the ask the input is kept for")
|
||||
@@ -72,29 +106,43 @@ func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
|
||||
if err := k.sweep(); err != nil {
|
||||
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
|
||||
}
|
||||
if k.used != nil {
|
||||
used, err := k.used()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
|
||||
}
|
||||
if used+int64(len(body)) > inputsCap {
|
||||
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
|
||||
meshwants.StateInputs, inputsCap>>20, used, len(body))
|
||||
}
|
||||
}
|
||||
digest := digestOf(body)
|
||||
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
|
||||
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
askKey := inputsAskPrefix + keys.EscapeName(ask)
|
||||
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
have, err := k.vals.KV.Get(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if have != nil {
|
||||
return digest, nil
|
||||
}
|
||||
if k.used != nil {
|
||||
used, err := k.used()
|
||||
if err == nil && used+int64(len(body)) > inputsCap {
|
||||
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
|
||||
buckets.InputsState, inputsCap>>20, used, len(body))
|
||||
} else if err != nil {
|
||||
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
|
||||
}
|
||||
if err != nil {
|
||||
_ = k.vals.KV.Delete(askKey)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
|
||||
// input a sweep deleted, changes nothing.
|
||||
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
|
||||
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
|
||||
func (k *inputKeeper) ended(ask string) error {
|
||||
if ask == "" {
|
||||
return nil
|
||||
@@ -104,66 +152,165 @@ func (k *inputKeeper) ended(ask string) error {
|
||||
if err != nil || e == nil {
|
||||
return err
|
||||
}
|
||||
if err := k.vals.KV.Delete(string(e.Value)); err != nil {
|
||||
digest, _ := readAsk(e.Value)
|
||||
if err := k.vals.KV.Delete(askKey); err != nil {
|
||||
return err
|
||||
}
|
||||
return k.vals.KV.Delete(askKey)
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if named[digest] {
|
||||
return nil // another running ask still names it
|
||||
}
|
||||
return k.deleteInput(digest)
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
|
||||
// named is the inputs some ask's key names now.
|
||||
func (k *inputKeeper) named() (map[string]bool, error) {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
named := map[string]bool{}
|
||||
for _, key := range all {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
continue
|
||||
}
|
||||
o, err := k.vals.KV.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o != nil {
|
||||
d, _ := readAsk(o.Value)
|
||||
named[d] = true
|
||||
}
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
|
||||
// ends its object. A key already gone leaves nothing to remove here; the start and assigner_inputs_drop remove what a key left.
|
||||
func (k *inputKeeper) deleteInput(key string) error {
|
||||
e, err := k.vals.KV.Get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
return err
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
if ref, ok := stateobjects.ReadRef(e.Value); ok {
|
||||
if err := k.vals.Objects.Delete(ref.Object); err != nil {
|
||||
return fmt.Errorf("the object %s of %s: %w", ref.Object, key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropKeyGone removes every object of the state whose key is gone and answers their count and bytes (A-465).
|
||||
func (k *inputKeeper) dropKeyGone() (dropped []stateobjects.Kept, bytes int64, err error) {
|
||||
kept, err := k.vals.Kept()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
names, err := k.vals.DropKeyGone()
|
||||
gone := map[string]bool{}
|
||||
for _, n := range names {
|
||||
gone[n] = true
|
||||
}
|
||||
for _, o := range kept {
|
||||
if gone[o.Name] {
|
||||
dropped = append(dropped, o)
|
||||
bytes += o.Size
|
||||
}
|
||||
}
|
||||
return dropped, bytes, err
|
||||
}
|
||||
|
||||
// dropAtStart removes the objects whose key is gone at the program's start, said with their count and bytes (A-465). An object
|
||||
// store not on the bus yet is said and left.
|
||||
func (k *inputKeeper) dropAtStart() {
|
||||
dropped, bytes, err := k.dropKeyGone()
|
||||
if err != nil {
|
||||
k.logf("inputs: the objects of deleted keys are not all removed at the start (%d objects, %d bytes removed): %v", len(dropped), bytes, err)
|
||||
return
|
||||
}
|
||||
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
|
||||
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
|
||||
// input is gone (A-465b).
|
||||
func (k *inputKeeper) sweep() error {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
var inputs, asks []string
|
||||
type askAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var asks []askAt
|
||||
var inputs []string
|
||||
for _, key := range all {
|
||||
if strings.HasPrefix(key, inputsAskPrefix) {
|
||||
asks = append(asks, key)
|
||||
} else {
|
||||
inputs = append(inputs, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(inputs) // the write time leads the key, so the oldest come first
|
||||
if len(inputs) > inputsSwept {
|
||||
inputs = inputs[:inputsSwept]
|
||||
}
|
||||
for _, key := range inputs {
|
||||
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
|
||||
at, perr := time.Parse(inputsTime, t)
|
||||
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
|
||||
continue
|
||||
}
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
for _, key := range asks {
|
||||
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
|
||||
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
errs = append(errs, err)
|
||||
e, err := k.vals.KV.Get(key)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
d, at := readAsk(e.Value)
|
||||
if !isDigest(d) {
|
||||
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
|
||||
errs = append(errs, k.vals.KV.Delete(key))
|
||||
}
|
||||
}
|
||||
asks = append(asks, askAt{key, at})
|
||||
continue
|
||||
}
|
||||
if !isDigest(key) {
|
||||
t, _, _ := strings.Cut(key, ".")
|
||||
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
continue
|
||||
}
|
||||
inputs = append(inputs, key)
|
||||
}
|
||||
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
|
||||
if len(asks) > inputsSwept {
|
||||
asks = asks[:inputsSwept]
|
||||
}
|
||||
for _, a := range asks {
|
||||
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
|
||||
errs = append(errs, k.vals.KV.Delete(a.key))
|
||||
}
|
||||
}
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, err)...)
|
||||
}
|
||||
for _, key := range inputs {
|
||||
if !named[key] {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// keyOf is the key of the input of the digest, or none.
|
||||
// keyOf is the key of the input of the digest, or none: the digest itself.
|
||||
func (k *inputKeeper) keyOf(digest string) (string, error) {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
if !isDigest(digest) {
|
||||
return "", nil
|
||||
}
|
||||
e, err := k.vals.KV.Get(digest)
|
||||
if err != nil || e == nil {
|
||||
return "", err
|
||||
}
|
||||
slices.Sort(all)
|
||||
for _, key := range all {
|
||||
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
|
||||
@@ -173,7 +320,7 @@ func (k *inputKeeper) body(digest string) ([]byte, error) {
|
||||
}
|
||||
key, err := k.keyOf(digest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
|
||||
@@ -241,7 +388,7 @@ func (o *inputObjects) open() (stateobjects.Objects, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
|
||||
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
|
||||
}
|
||||
@@ -292,7 +439,7 @@ func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, lo
|
||||
return nil
|
||||
}
|
||||
objects := &inputObjects{js: js, module: module}
|
||||
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
|
||||
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
|
||||
used := func() (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
@@ -333,12 +480,21 @@ func inputsRead(k *inputKeeper) stdio.Tool {
|
||||
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
|
||||
for _, key := range all {
|
||||
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
|
||||
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
|
||||
lastAsk[d] = at.Format(inputsTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
out := []inputSaid{}
|
||||
for _, key := range all {
|
||||
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
|
||||
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
|
||||
d := key
|
||||
if !isDigest(key) || (want != "" && d != want) {
|
||||
continue
|
||||
}
|
||||
t := lastAsk[key]
|
||||
var text string
|
||||
if found, err := k.vals.Get(key, &text); err != nil || !found {
|
||||
continue
|
||||
@@ -352,26 +508,74 @@ func inputsRead(k *inputKeeper) stdio.Tool {
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
return map[string]any{"inputs": out}, nil
|
||||
gone := []map[string]any{}
|
||||
var goneBytes int64
|
||||
answer := map[string]any{"inputs": out}
|
||||
kept, err := k.vals.Kept()
|
||||
if err != nil {
|
||||
// never an empty list for objects that could not be read (A-465)
|
||||
answer["key-gone-error"] = fmt.Sprintf("the objects of the state inputs cannot be listed, so key-gone objects are not known: %v", err)
|
||||
return answer, nil
|
||||
}
|
||||
for _, o := range kept {
|
||||
if o.KeyGone {
|
||||
gone = append(gone, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
|
||||
goneBytes += o.Size
|
||||
}
|
||||
}
|
||||
answer["key-gone"], answer["key-gone-bytes"] = gone, goneBytes
|
||||
return answer, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
|
||||
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
|
||||
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
|
||||
// inputsDropTool is the tool that removes the objects whose key is gone.
|
||||
const inputsDropTool = "assigner_inputs_drop"
|
||||
|
||||
// inputsDrop is the tool assigner_inputs_drop (A-465): a hand act, recorded first, that removes the objects of the state inputs
|
||||
// whose key is gone now and answers what it removed.
|
||||
func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: inputsDropTool, Description: "Remove the objects of the assigner's state inputs whose key is gone now (design 47 G A-465), " +
|
||||
"and answer which and how many bytes. An object a key names is never removed. Recorded as a hand act; assigner_inputs lists them as key-gone.",
|
||||
Input: schema(map[string]string{"why": "why the objects of deleted keys are dropped now"}, []string{"why"}), Run: func(args map[string]any) (any, error) {
|
||||
why := strings.TrimSpace(arg(args, "why"))
|
||||
if why == "" {
|
||||
return nil, errors.New("say why the objects of deleted keys are dropped")
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: inputsDropTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "inputs-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was removed: %w", err)
|
||||
}
|
||||
dropped, bytes, err := k.dropKeyGone()
|
||||
out := []map[string]any{}
|
||||
for _, o := range dropped {
|
||||
out = append(out, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
|
||||
}
|
||||
return map[string]any{"dropped": out, "count": len(out), "bytes": bytes, "hand-act": taken.Log}, err
|
||||
}}
|
||||
}
|
||||
|
||||
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
|
||||
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
|
||||
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
|
||||
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
|
||||
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var e buildstep.StepEnded
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if runEnded != nil {
|
||||
if err := runEnded(e); err != nil {
|
||||
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := k.ended(e.Ask); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if ends != nil {
|
||||
ends.heard(e)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -14,6 +16,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
)
|
||||
@@ -22,6 +25,8 @@ type kvMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
// delErr, when set, fails the next Delete once.
|
||||
delErr error
|
||||
}
|
||||
|
||||
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
@@ -56,6 +61,10 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error {
|
||||
func (k *kvMem) Delete(key string) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if err := k.delErr; err != nil {
|
||||
k.delErr = nil
|
||||
return err
|
||||
}
|
||||
delete(k.vals, key)
|
||||
return nil
|
||||
}
|
||||
@@ -71,8 +80,9 @@ func (k *kvMem) Keys() ([]string, error) {
|
||||
}
|
||||
|
||||
type objectsMem struct {
|
||||
mu sync.Mutex
|
||||
objs map[string][]byte
|
||||
mu sync.Mutex
|
||||
objs map[string][]byte
|
||||
listErr error // List fails with it
|
||||
}
|
||||
|
||||
func (*objectsMem) MostBytes() int { return 0 }
|
||||
@@ -91,8 +101,24 @@ func (o *objectsMem) Get(name string) ([]byte, error) {
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
func (o *objectsMem) Delete(string) error { return nil }
|
||||
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
|
||||
func (o *objectsMem) Delete(name string) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
delete(o.objs, name)
|
||||
return nil
|
||||
}
|
||||
func (o *objectsMem) List() ([]stateobjects.Object, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.listErr != nil {
|
||||
return nil, o.listErr
|
||||
}
|
||||
var out []stateobjects.Object
|
||||
for n, d := range o.objs {
|
||||
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
|
||||
type lineMem struct {
|
||||
@@ -228,4 +254,250 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
|
||||
if _, err := k.body(d2); err == nil {
|
||||
t.Error("an input a day old is still kept at the next ask")
|
||||
}
|
||||
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
|
||||
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
delErr error
|
||||
runErr error
|
||||
naked bool
|
||||
runs int
|
||||
deleted bool
|
||||
}{
|
||||
{name: "the run's end before the input's delete", runs: 1, deleted: true},
|
||||
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
|
||||
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
|
||||
k.vals.KV.(*kvMem).delErr = c.delErr
|
||||
runs := 0
|
||||
runEnded := func(e buildstep.StepEnded) error {
|
||||
runs++
|
||||
if _, err := k.body(d); err != nil {
|
||||
t.Errorf("the run's end is written after the input's delete (%d)", runs)
|
||||
}
|
||||
return c.runErr
|
||||
}
|
||||
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
|
||||
m := &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, runEnded, m, t.Logf)
|
||||
if m.naked != c.naked || m.acked == c.naked {
|
||||
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
|
||||
}
|
||||
if m.naked && c.delErr != nil { // the redelivery
|
||||
m = &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
|
||||
}
|
||||
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
|
||||
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
|
||||
// too; the objects of keys another part deleted are removed by dropKeyGone and by the tool assigner_inputs_drop, which lists
|
||||
// them as key-gone first, and never removes an object a key names.
|
||||
func TestAnEndedInputEndsItsObjectAndTheDropToolRemovesWhatAKeyLeft(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
big := func(b byte) []byte { return bytes.Repeat([]byte{b}, 2<<20) }
|
||||
if _, err := k.keep("ask-one", big('a')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.keep("ask-two", big('b')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs.objs) != 2 {
|
||||
t.Fatalf("objects = %d, want 2", len(objs.objs))
|
||||
}
|
||||
if err := k.ended("ask-one"); err != nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("the step-ended left %d objects (%v), want 1", len(objs.objs), err)
|
||||
}
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask-three", big('c')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs.objs) != 1 {
|
||||
t.Fatalf("after the sweep of the day-old input %d objects, want only the new one", len(objs.objs))
|
||||
}
|
||||
// An object whose key another part deleted is listed as key-gone and removed by the tool; a live one stays.
|
||||
keys, _ := k.vals.KV.Keys()
|
||||
for _, key := range keys {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
live := []byte(`{"heads":"x"}`)
|
||||
if _, err := k.keep("ask-four", append(live, make([]byte, 2<<20)...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := inputsRead(k).Run(map[string]any{})
|
||||
if err != nil || len(list.(map[string]any)["key-gone"].([]map[string]any)) != 1 {
|
||||
t.Fatalf("assigner_inputs key-gone = %+v %v, want the one object of the deleted key", list, err)
|
||||
}
|
||||
ask := func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"answer":{"log":"hand-acts"}}`), nil
|
||||
}
|
||||
if _, err := inputsDrop(k, ask).Run(map[string]any{}); err == nil {
|
||||
t.Error("dropped with no reason")
|
||||
}
|
||||
out, err := inputsDrop(k, ask).Run(map[string]any{"why": "a key left its object"})
|
||||
if err != nil || out.(map[string]any)["count"] != 1 || len(objs.objs) != 1 {
|
||||
t.Fatalf("assigner_inputs_drop = %+v %v, %d objects left: want one removed and the live one kept", out, err, len(objs.objs))
|
||||
}
|
||||
// The start drops what a key left, said with its count and bytes.
|
||||
if _, err := k.keep("ask-five", live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range keysOf(t, k) {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
_ = k.vals.KV.Delete(key)
|
||||
}
|
||||
}
|
||||
var logged []string
|
||||
k.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
before := len(objs.objs)
|
||||
k.dropAtStart()
|
||||
if len(objs.objs) != 0 || before == 0 || len(logged) != 1 || !strings.Contains(logged[0], fmt.Sprintf("%d objects of deleted keys removed at the start", before)) || !strings.Contains(logged[0], " bytes") {
|
||||
t.Errorf("the start left %d of %d objects and said %q: want every key-gone object removed and its count and bytes said", len(objs.objs), before, logged)
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(t *testing.T, k *inputKeeper) []string {
|
||||
t.Helper()
|
||||
keys, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Design 47 G A-465: assigner_inputs says why key-gone objects are not known when the objects cannot be listed, and never
|
||||
// answers them as none.
|
||||
func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
|
||||
objs := &objectsMem{objs: map[string][]byte{}, listErr: errors.New("the object store does not answer")}
|
||||
k := testKeeper(objs, time.Now)
|
||||
out, err := inputsRead(k).Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if msg, _ := m["key-gone-error"].(string); !strings.Contains(msg, "does not answer") {
|
||||
t.Errorf("answer %+v: want key-gone-error naming the failure", m)
|
||||
}
|
||||
if _, has := m["key-gone"]; has {
|
||||
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
|
||||
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
|
||||
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
|
||||
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
body := bytes.Repeat([]byte{'n'}, 2<<20)
|
||||
var digests []string
|
||||
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
|
||||
d, err := k.keep(ask, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests = append(digests, d)
|
||||
}
|
||||
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
|
||||
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
|
||||
}
|
||||
if _, err := k.vals.KV.Get(digests[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
|
||||
}
|
||||
for _, key := range keysOf(t, k) {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
|
||||
t.Errorf("a key %q beside the one input", key)
|
||||
}
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
|
||||
t.Errorf("the ask does not name the digest it uses: %v", e)
|
||||
}
|
||||
for _, ask := range []string{"ask-1", "ask-2"} {
|
||||
if err := k.ended(ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
|
||||
}
|
||||
}
|
||||
if err := k.ended("ask-3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
|
||||
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
|
||||
}
|
||||
// a day after its last write, a renewed write counted
|
||||
if _, err := k.keep("ask-4", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-5", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-6", []byte("other")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Errorf("an input written again 20 hours ago was swept: %v", err)
|
||||
}
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask-7", []byte("another")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
|
||||
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
|
||||
}
|
||||
// an input of the old form goes a day after the time its key carries, with its object, never earlier
|
||||
old := "20261117T090000Z." + digestOf([]byte("old"))
|
||||
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
|
||||
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.sweep(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
|
||||
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(recent); e == nil {
|
||||
t.Error("an input of the old form under a day old was deleted early")
|
||||
}
|
||||
// a new content the cap refuses says the fill; the same content, already kept, needs no room
|
||||
full := newTestKeeperWithFill(objs, now, inputsCap-10)
|
||||
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
|
||||
t.Fatalf("a content that fits is refused: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
|
||||
t.Errorf("the refusal does not name the fill: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
|
||||
t.Errorf("the content already kept is refused at the cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
k.used = func() (int64, error) { return used, nil }
|
||||
return k
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,9 +9,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
@@ -22,6 +25,7 @@ import (
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -30,7 +34,6 @@ import (
|
||||
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
@@ -40,12 +43,16 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// composeCallBound is how long the program's calls wait for an answer (main).
|
||||
const composeCallBound = 3 * time.Minute
|
||||
|
||||
func main() {
|
||||
if os.Getenv(process.BrokerFileEnv) == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
@@ -56,6 +63,10 @@ func main() {
|
||||
}
|
||||
return
|
||||
}
|
||||
// A call waits up to composeCallBound for its answer: compose-candidate composes a whole revision in shadow on each node and
|
||||
// answered in about 34 s on g14 and shanks (2026-11-16 23:27), past the SDK's 30 s, so every composition step could not run.
|
||||
// A responder that is absent still answers at once. To be replaced by a bound per verb from the seat's definition.
|
||||
process.CallTimeout = composeCallBound
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
@@ -69,7 +80,6 @@ func main() {
|
||||
var readBuckets = []string{
|
||||
buckets.Grants,
|
||||
buckets.Have,
|
||||
buckets.Provisions,
|
||||
}
|
||||
|
||||
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
|
||||
@@ -82,11 +92,11 @@ func ownBuckets() []string {
|
||||
return []string{buckets.Asks(), buckets.Differences(),
|
||||
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
|
||||
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
|
||||
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
|
||||
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()}
|
||||
}
|
||||
|
||||
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
|
||||
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
|
||||
// engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `<node>`
|
||||
// whose value is the words and the time of the last manifest-reads-changed heard of it: what assigner_engine_reads lists.
|
||||
const engineReadsState = "engine-reads"
|
||||
|
||||
// engineReadsBucket is that state's bucket.
|
||||
@@ -126,11 +136,10 @@ var keptStates = []announce.State{
|
||||
{Name: copiesState, Readable: true},
|
||||
{Name: writtenState, Readable: true},
|
||||
{Name: buckets.WaitsState, Readable: true},
|
||||
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
|
||||
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
|
||||
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
|
||||
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
|
||||
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
|
||||
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
|
||||
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
|
||||
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
|
||||
{Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777)
|
||||
}
|
||||
|
||||
// kvSet is every bucket the assigner opened, by name.
|
||||
@@ -159,12 +168,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == buckets.Provisions && err != nil {
|
||||
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
|
||||
// until then no want waits for a provision, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.PoliciesBucket() && err != nil {
|
||||
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
|
||||
// person's policy is read, so the controller's beside each composition holds, as before.
|
||||
@@ -178,8 +181,8 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
continue
|
||||
}
|
||||
if name == engineReadsBucket() && err != nil {
|
||||
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
|
||||
// assigner_engine_reads lists nothing.
|
||||
// Without it the words each manifest-reads-changed says are kept in memory only, and assigner_engine_reads
|
||||
// lists nothing.
|
||||
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
@@ -188,27 +191,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err)
|
||||
continue
|
||||
}
|
||||
if name == pinsBucket() && err != nil {
|
||||
// New with the pins (design 47 G §579): without it no pin is kept and no revision is removed.
|
||||
logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == buckets.Waits() && err != nil {
|
||||
// New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged
|
||||
// as before.
|
||||
logf("the bucket %s cannot be opened (%v): no outside wait is kept until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == buckets.DeliveredAgain() && err != nil {
|
||||
// New with A-250: without it assigner_delivered_again is not taken until the assigner starts again.
|
||||
logf("the bucket %s cannot be opened (%v): no version is said delivered again until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == buckets.DeliveryWaits() && err != nil {
|
||||
// New with A-246: without it no dependent waits for its dependency's delivery until the assigner starts again.
|
||||
logf("the bucket %s cannot be opened (%v): no delivery wait is kept until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == buckets.PutWaits() && err != nil {
|
||||
// New with A-211: without it no put's wait is said until the assigner starts again.
|
||||
logf("the bucket %s cannot be opened (%v): no put's wait is said until the assigner starts again", name, err)
|
||||
@@ -237,8 +225,8 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case plc.SeatHoldersBucket(), buckets.Provisions, assign.PoliciesBucket(), assign.FindingsBucket(),
|
||||
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket():
|
||||
case plc.SeatHoldersBucket(), assign.PoliciesBucket(), assign.FindingsBucket(),
|
||||
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs():
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets(), name) || name == buckets.Grants
|
||||
@@ -294,10 +282,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The inputs of the composition steps it asks a build seat to run: the seat mesh-wants' verb input is answered here in chunks
|
||||
// of the core connection, of any size (design 47 G §354, §712, A-237).
|
||||
inputs := inputsOver(kvs[buckets.Inputs()], js, account.Module, logger.Printf)
|
||||
// The seat list (seatlist.go), followed in follow over the connection and the hub: the verb input's callers are checked
|
||||
// against the seat's definition it holds (A-186).
|
||||
gate := seatlist.NewPart("assigner", seatListAgainCallers)
|
||||
// The seat list (seatlist.go), followed in follow over the connection and the hub.
|
||||
gate := seatlist.NewPart("assigner")
|
||||
if inputs != nil {
|
||||
inputs.dropAtStart() // the objects of keys another part deleted (A-465)
|
||||
stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("answering the verb input: %w", err)
|
||||
@@ -378,14 +366,11 @@ func watched() []string {
|
||||
buckets.Differences(),
|
||||
buckets.Wants(),
|
||||
assign.PoliciesBucket(),
|
||||
buckets.Provisions,
|
||||
plc.SeatHoldersBucket(),
|
||||
engineReadsBucket(),
|
||||
writtenBucket(),
|
||||
buckets.Waits(),
|
||||
buckets.PutWaits(),
|
||||
buckets.DeliveryWaits(),
|
||||
buckets.DeliveredAgain(), // its watch wakes a pass at once on an ask of assigner_delivered_again (A-250)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,8 +378,6 @@ func watched() []string {
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, inputs *inputKeeper, gate *seatlist.Part, logger *log.Logger, getenv func(string) string) error {
|
||||
w := newWorld()
|
||||
// The callers' seat entries are resolved from the assigner's own seat-holders as it last read them (A-298).
|
||||
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return w.read(plc.SeatHoldersBucket()), nil }))
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
select {
|
||||
@@ -404,16 +387,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
errs := make(chan error, len(watched())+2)
|
||||
settled := make(chan struct{}, len(watched())+2)
|
||||
// What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read
|
||||
// from the node's record where the engine does not answer (the fallback, said in the journal).
|
||||
reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf}
|
||||
if bus != nil {
|
||||
reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1)
|
||||
reads.Once = bus.CallOn // its fault keeps the fallback at once while the kept call asks on (ADR 0668 §3)
|
||||
}
|
||||
// What each node's engine reads (design 47 H-A-104): the words of its manifest-reads-changed, kept in the state
|
||||
// engine-reads and read back from its watch, so a restart loses none; no verb is asked.
|
||||
reads := &engineReadsKept{changed: wake}
|
||||
if kv := kvs[engineReadsBucket()]; kv != nil {
|
||||
reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
reads.Delete = func(node string) error { return kv.Delete(ctx, node) }
|
||||
reads.put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
}
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
@@ -449,7 +427,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
value = e.Value()
|
||||
}
|
||||
if name == engineReadsBucket() {
|
||||
reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed
|
||||
if err := reads.take(e.Key(), value, value == nil); err != nil { // it wakes a pass when the words changed
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
@@ -516,17 +496,35 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
|
||||
}
|
||||
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
|
||||
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
|
||||
// §389). Registered before the hub starts.
|
||||
ends, away := &runEnds{}, &awayNodes{}
|
||||
if inputs != nil {
|
||||
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
|
||||
// for it (A-240). The machines a composition step does not compose are the watchman's machine-away, heard once, in the world (§64, §389).
|
||||
// The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends
|
||||
// the run in its key.
|
||||
var runs *compositionRuns
|
||||
if kv := kvs[buckets.CompositionRuns()]; kv != nil {
|
||||
set, err := runSettingsOf(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf)
|
||||
if js != nil {
|
||||
runs.say = wantsPublisher(ctx, js, thisNode)
|
||||
}
|
||||
if bus != nil {
|
||||
runs.drop = dropOver(bus.CallKept)
|
||||
}
|
||||
// The build seats' ask-taken and step-progress begin a run's clock (§720, A-466q).
|
||||
if err := hearRunBounds(hub, runs, logger.Printf); err != nil {
|
||||
logger.Printf("the build seats' ask-taken and step-progress are not heard, so a composition run is ended by no bound: %v", err)
|
||||
}
|
||||
}
|
||||
for subject, isAway := range map[string]bool{meshwatchdogs.SubjectMachineAwayAll: true, meshwatchdogs.SubjectMachineBackAll: false} {
|
||||
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPresence(away, isAway, m, logger.Printf) }); err != nil {
|
||||
logger.Printf("%s is not heard, so a composition step asks a machine that sleeps: %v", subject, err)
|
||||
if inputs != nil {
|
||||
var runEnded func(buildstep.StepEnded) error
|
||||
if runs != nil && bus != nil {
|
||||
read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) }
|
||||
runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) }
|
||||
}
|
||||
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
|
||||
@@ -558,7 +556,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), buckets.Provisions, plc.SeatHoldersBucket()} {
|
||||
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), plc.SeatHoldersBucket()} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
@@ -587,7 +585,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
reads: reads, seats: gate}
|
||||
reads: reads, seats: gate, askForFile: getenv("MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE"), listed: iv.listed}
|
||||
p.askOperatorFor() // the setting ask-operator-for said once at the start (design 47 G A-425a)
|
||||
// The setting tier-bound (design 47 H §751) and the one timer that wakes the pass at a tier's deadline (H-A-56).
|
||||
tierBound, err := tierBoundSetting(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.tierBound, p.tierWake = tierBound, wake
|
||||
// The wait on an outside cause (outside.go, design 47 G §706): the keeper's three events say when to judge again, and the
|
||||
// seat mesh-wants' wait-held and wait-ended are said; registered before the hub starts.
|
||||
if bus != nil && kvs[buckets.Waits()] != nil {
|
||||
@@ -607,39 +612,38 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
logger.Printf("%s is not heard, so a wait on an outside cause is judged again only when its check changes: %v", subject, err)
|
||||
}
|
||||
}
|
||||
p.outside.readStart()
|
||||
}
|
||||
if js != nil {
|
||||
p.publishWants = wantsPublisher(ctx, js, thisNode)
|
||||
if iv != nil && iv.h != nil {
|
||||
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)
|
||||
p.graph = &graphKeeper{objects: objects, halves: iv.h.views}
|
||||
// The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes
|
||||
// one (design 47 H-A-259).
|
||||
retry, err := keepingRetry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, retry: retry,
|
||||
reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)}
|
||||
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
|
||||
return p.keepCandidate(ctx, iv.call, ev)
|
||||
}
|
||||
iv.candidates.Store(&keep)
|
||||
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
|
||||
p.entries = meshcatalogclient.NewEntries(iv.h.call)
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
|
||||
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
|
||||
// on the hub, registered before it starts.
|
||||
if kv := kvs[pinsBucket()]; kv != nil {
|
||||
k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf}
|
||||
if err := k.made(); err != nil {
|
||||
logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err)
|
||||
}
|
||||
if err := hearPins(hub, k); err != nil {
|
||||
logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err)
|
||||
} else {
|
||||
p.graph.pins = k
|
||||
k.watchUnpins(kv)
|
||||
}
|
||||
entryAsks, err := settingFile(getenv("MESH_ASSIGNER_ENTRY_ASKS_FILE"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.entries = entriesOf(iv.h.branches, iv.h.call, entryAsks, iv.writer, wake)
|
||||
askAgainOnReconnect(ctx, nc, p.entries)
|
||||
// The revisions that are not current are removed when one ceases to be current and once now (revisions.go,
|
||||
// design 47 H §764, H-A-86).
|
||||
p.graph.revisions = &revisionRemover{ctx: ctx, objects: objects, inUse: inUseOf(w), logf: logger.Printf}
|
||||
// The start's removal of the revisions not current folds into the first pass's keeping (H-A-261).
|
||||
}
|
||||
}
|
||||
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
|
||||
// Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it
|
||||
// is not there yet, a node is asked again only when it comes back online.
|
||||
// The node-engine seat's manifest-reads-changed, for one node (design 47 H-A-104): its words are written into the state
|
||||
// engine-reads. Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes.
|
||||
if js != nil {
|
||||
// The teller's ask-ended (issue 1966): an ask ended at the channel without an answer is written again; registered
|
||||
// before hearReadsChanged starts the hub.
|
||||
@@ -652,20 +656,18 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// once the first registrations are written (§95, §353).
|
||||
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
|
||||
p.ends.trunk = w.trunkName
|
||||
p.ends.releasedOf = w.releasedOf // a version released before its pipeline passed is taken at the pass (H-A-490)
|
||||
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
|
||||
err := hearEnds(hub, p.ends)
|
||||
if err == nil && bus != nil {
|
||||
err = followRegistrations(ctx, hub, bus.Call, p.ends)
|
||||
if err := hearEnds(hub, p.ends); err != nil {
|
||||
logger.Printf("a pipeline's end is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
|
||||
// The registrations come from the catalog half's one follower, its one handler of registration-changed (H-A-270 (3)).
|
||||
if iv == nil || iv.h == nil || !iv.h.followRegistrations(p.ends.registered) {
|
||||
logger.Printf("the registrations are not followed, so the wants keep the trunk-only form until the assigner starts again: the catalog half is not followed")
|
||||
}
|
||||
}
|
||||
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
|
||||
hearReleases(ctx, js, w, wake, findings, logger.Printf) // version-released, machine-away and machine-back
|
||||
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
|
||||
if err := hearTaken(hub, &w.taken, wake, logger.Printf); err != nil {
|
||||
logger.Printf("the indexer's take-ins are not heard, so an entry held by an order edge waits until the assigner starts again: %v", err)
|
||||
}
|
||||
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
|
||||
hearRunning(ctx, js, w, wake, findings, logger.Printf)
|
||||
}
|
||||
@@ -673,15 +675,64 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, call: bus.CallKeptOn}, keeper: inputs, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
|
||||
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
|
||||
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
|
||||
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
|
||||
} else {
|
||||
logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep)
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
|
||||
askBound, err := composeAskBound()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, callFile: bus.CallFileKept,
|
||||
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
|
||||
takenBound, err := askTakenBound()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
|
||||
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume,
|
||||
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
|
||||
takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })}
|
||||
if e, ok := bus.(emitting); ok {
|
||||
steps.taken = e.State(process.TakenState)
|
||||
}
|
||||
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
|
||||
runs.moved.Store(&move)
|
||||
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
|
||||
// once the values that stand are read (§719, §720).
|
||||
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
|
||||
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
// The runs' read and drop, the seat mesh-wants' verbs compositions and compositions-drop (§728, A-469f), answered here
|
||||
// where the runs are kept and the readers' ends said; the drop's hand act recorded through the seat mesh-hand-acts first.
|
||||
if e, ok := bus.(emitting); ok {
|
||||
record := func(q meshhandactsclient.Request) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
|
||||
return err
|
||||
}
|
||||
verbs := compositionVerbs(runs, steps, record)
|
||||
maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404)
|
||||
if p.graph != nil {
|
||||
maps.Copy(verbs, keepNowVerb(ctx, p)) // the keeping by hand (§813 (2), H-A-417)
|
||||
}
|
||||
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs)
|
||||
if err != nil {
|
||||
logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err)
|
||||
} else {
|
||||
go func() { <-ctx.Done(); stopVerbs() }()
|
||||
}
|
||||
}
|
||||
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
|
||||
if p.ends != nil {
|
||||
left := func(pipeline string) error {
|
||||
_, err := runs.leave(pipeline, steps.leave)
|
||||
return err
|
||||
}
|
||||
p.ends.left.Store(&left)
|
||||
}
|
||||
if steps.taken == nil {
|
||||
logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState)
|
||||
} else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil {
|
||||
logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err)
|
||||
}
|
||||
}
|
||||
if bus != nil {
|
||||
@@ -746,48 +797,51 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) erro
|
||||
}
|
||||
}
|
||||
|
||||
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: acknowledges it, keeps what it says
|
||||
// of the node (a statement older than it does not stand) and asks the node again.
|
||||
func readsChangedHandler(reads *enginereads.Keeper, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: writes the words it says of its node
|
||||
// into the state engine-reads, then acknowledges it; an event not written is not acknowledged, so the bus delivers it again.
|
||||
func readsChangedHandler(reads *engineReadsKept, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
return func(m jetstream.Msg) {
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
if ours && err == nil {
|
||||
var e nodeengine.ManifestReadsChangedEvent
|
||||
if e, err = enginereads.DecodeChanged(m.Data()); err == nil {
|
||||
if err = reads.heard(e); err != nil {
|
||||
logf("%s of %s: %v", enginereads.Event, node, err)
|
||||
if nerr := m.Nak(); nerr != nil {
|
||||
logf("not acknowledging %s: %v", m.Subject(), nerr)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if ours && err != nil {
|
||||
logf("%s holds what is not %s, and is not kept: %v", m.Subject(), enginereads.Event, err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
switch {
|
||||
case !ours:
|
||||
case err != nil:
|
||||
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
|
||||
reads.AskNode(node, enginereads.Event+" that does not read")
|
||||
default:
|
||||
// What the event says is kept first: a statement older than it does not stand while the node is asked again.
|
||||
if e, derr := enginereads.DecodeChanged(m.Data()); derr == nil {
|
||||
reads.HeardChanged(e)
|
||||
}
|
||||
reads.AskNode(node, enginereads.Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's one hub of its durable
|
||||
// consumer on EVENTS, and asks the node it names again; the hub also hands the indexer's views to the followers of
|
||||
// consumer on EVENTS, and keeps the words of the node it names; the hub also hands the indexer's views to the followers of
|
||||
// indexerViews, and starts once both are registered.
|
||||
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *enginereads.Keeper, findings assign.Findings, live func(), logf func(string, ...any)) {
|
||||
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *engineReadsKept, findings assign.Findings, live func(), logf func(string, ...any)) {
|
||||
account := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
w := notHeardWriter(findings, logf)
|
||||
handle := readsChangedHandler(reads, logf)
|
||||
err := hub.Handle(enginereads.SubjectPrefix+"*", handle)
|
||||
// At each bind every node the assigner holds a statement of is asked again: an event said before its consumer existed
|
||||
// is not delivered to it (issue 1966 note 155: ace's bootstrapped statement stood; ADR 0668 §4).
|
||||
hub.OnBound(reads.AtBind(func() []string { return slices.Sorted(maps.Keys(reads.Statements())) }))
|
||||
if err == nil {
|
||||
err = hub.Start(ctx, js, account)
|
||||
}
|
||||
if err != nil {
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again (a node is asked again when it comes back online)", err)
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again, so no node's new words are kept", err)
|
||||
return
|
||||
}
|
||||
w.Heard(process.EventsStream)
|
||||
// The build seats' step-ended is a kept seat event, on its own stream and never on EVENTS: bound here, or no composition
|
||||
// run's end is heard and every composition step waits for ever (design 47 G §354, A-240; mesh-sdk EventHub.StartKept).
|
||||
logf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, account))
|
||||
if live != nil {
|
||||
live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive)
|
||||
}
|
||||
@@ -796,3 +850,35 @@ func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.
|
||||
hub.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
// entriesOf are the catalog entries the passer asks by fingerprint: the followed branches' own, so each catalog-revision asks
|
||||
// again an entry whose ask failed to reach the holder, at most asks times (the setting entry-asks, 0 its default), and the
|
||||
// assigner's finding writer writes entry-not-found and entry-unanswered of them (design 47 H, H-A-60). Each change wakes a pass.
|
||||
func entriesOf(b *meshcatalogclient.Branches, call meshcatalogclient.Call, asks int, w views.WireWriter, wake func()) *meshcatalogclient.Entries {
|
||||
if b == nil || b.Entries == nil {
|
||||
b = &meshcatalogclient.Branches{Entries: meshcatalogclient.NewEntries(call)}
|
||||
}
|
||||
b.Entries.Asks = asks
|
||||
if w == nil {
|
||||
b.Entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
|
||||
return b.Entries
|
||||
}
|
||||
b.WriteEntryFindings(w, func(meshcatalogclient.EntryRow) { wake() })
|
||||
return b.Entries
|
||||
}
|
||||
|
||||
// askAgainOnReconnect asks the entries again whose ask failed each time the assigner's link to the bus opens again (design 47
|
||||
// H, H-A-60), after whatever the connection already does then.
|
||||
func askAgainOnReconnect(ctx context.Context, nc process.Subscriber, e *meshcatalogclient.Entries) {
|
||||
conn, ok := nc.(*nats.Conn)
|
||||
if !ok || conn == nil {
|
||||
return
|
||||
}
|
||||
before := conn.Opts.ReconnectedCB
|
||||
conn.SetReconnectHandler(func(c *nats.Conn) {
|
||||
if before != nil {
|
||||
before(c)
|
||||
}
|
||||
e.AskAgain(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,8 +16,10 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +28,7 @@ import (
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
@@ -125,7 +128,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
kvs[buckets.Puts()].(*watchKV).closed = true
|
||||
kvs[buckets.Wants()].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner", seatListAgainCallers), log.New(io.Discard, "", 0), func(string) string { return "" })
|
||||
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner"), log.New(io.Discard, "", 0), func(string) string { return "" })
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
@@ -201,11 +204,57 @@ func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error)
|
||||
return nil, jetstream.ErrNoKeysFound
|
||||
}
|
||||
|
||||
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
|
||||
// raise a finding into the same map.
|
||||
var putIntoMu sync.Mutex
|
||||
|
||||
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
|
||||
func putInto(into map[string][]byte) func(mcseat.Finding) error {
|
||||
return func(f mcseat.Finding) error {
|
||||
raw, err := json.Marshal(f)
|
||||
putIntoMu.Lock()
|
||||
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
|
||||
putIntoMu.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// fakeWire is a views part's finding writer that hands each written finding's kind on.
|
||||
type fakeWire struct{ written chan string }
|
||||
|
||||
func (f fakeWire) WriteWire(_ context.Context, finding map[string]any) error {
|
||||
k, _ := finding["kind"].(string)
|
||||
f.written <- k
|
||||
return nil
|
||||
}
|
||||
func (fakeWire) Clear(context.Context, string, string, string, string) error { return nil }
|
||||
func (fakeWire) ModuleNode() (string, string) { return "mesh-delivery-assigner", "n1" }
|
||||
|
||||
// H-A-60: the assigner's entries are asked again to the setting entry-asks, then written through its finding writer as the
|
||||
// finding entry-unanswered.
|
||||
func TestAnUnreachedEntryEndsInTheAssignersFindingEntryUnanswered(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
unreached := func(string, any) (json.RawMessage, error) { return nil, errors.New("no responders") }
|
||||
w := fakeWire{written: make(chan string, 4)}
|
||||
e := entriesOf(nil, unreached, 1, w, func() {})
|
||||
if e.Asks != 1 {
|
||||
t.Fatalf("entry-asks %d, want 1 from the setting", e.Asks)
|
||||
}
|
||||
e.Need(ctx, "src2:u")
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case kind := <-w.written:
|
||||
if kind != meshcatalogclient.EntryUnansweredKind {
|
||||
t.Fatalf("the finding written is %q, want %s", kind, meshcatalogclient.EntryUnansweredKind)
|
||||
}
|
||||
return
|
||||
default:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no finding entry-unanswered is written past the bound")
|
||||
}
|
||||
e.AskAgain(ctx)
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// The module claiming the seat node-engine is found by its served verbs, in the catalog's form `seat:<seat>.<verb>`
|
||||
// and in the bare form (live, 2026-11-16: every composition step said "holds no module claiming the seat node-engine").
|
||||
func TestTheNodeEngineEntryIsFoundByItsServedVerbs(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
serves []string
|
||||
found bool
|
||||
}{
|
||||
{"the catalog's seat form", []string{"seat:node-engine.compose", "seat:node-engine.acts"}, true},
|
||||
{"the bare form", []string{"node-engine.compose"}, true},
|
||||
{"another seat", []string{"seat:node-engine-x.compose", "seat:mesh-wants.ask"}, false},
|
||||
} {
|
||||
e, ok := nodeEngineEntry(map[string]meshcatalog.CatalogEntry{"other": {Serves: []string{"seat:mesh-builds.ask"}}, "engine": {Serves: c.serves}})
|
||||
if ok != c.found || (ok && e.Serves[0] != c.serves[0]) {
|
||||
t.Errorf("%s: found %v (%v), want %v", c.name, ok, e.Serves, c.found)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,43 @@ func (b *busObjects) Names(ctx context.Context) ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// PutDescribed stores data under name with the entry's module and fingerprint in its metadata, which the listing says (assigngraph.Describer,
|
||||
// design 47 H-A-259).
|
||||
func (b *busObjects) PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name, Metadata: map[string]string{entryModuleMeta: module, entryFpMeta: fp}}, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
// Described is every object with the module and fingerprint its metadata says, in one listing.
|
||||
func (b *busObjects) Described(ctx context.Context) ([]assigngraph.Described, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos, err := o.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return []assigngraph.Described{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]assigngraph.Described, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
out = append(out, assigngraph.Described{Name: i.Name, Module: i.Metadata[entryModuleMeta], Fp: i.Metadata[entryFpMeta]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The metadata keys of an entry object's module and fingerprint.
|
||||
const (
|
||||
entryModuleMeta = "module"
|
||||
entryFpMeta = "fp"
|
||||
)
|
||||
|
||||
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
|
||||
func (b *busObjects) Delete(ctx context.Context, name string) error {
|
||||
o, err := b.open(ctx)
|
||||
|
||||
@@ -1,296 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The three order edges of a delivery that crosses a form change (design 47 G §298, A-245, A-258, A-259, A-260). Each is an acting wait of the
|
||||
// assigner, kept in the state `waits` under `order.<node>.<module>.<part>` beside the acting waits, in the step placing, said as wait-held
|
||||
// when a want is held for it and wait-ended (ran, released, replaced) when it moves; assigner_waits reads them and assigner_release with
|
||||
// how `now` releases them, as it does an acting wait. The holders of the seats are read from the seat holders the assignments compose
|
||||
// (assignment.SeatHoldersOf), never named:
|
||||
// - the node-backup holder's and the bus module's (the holder of the seat mesh-broker) new versions run before a module whose kept data's
|
||||
// form changes is moved, since the restore point needs both (OrderBackup);
|
||||
// - at a revert a node's node-engine goes back last on that node, after every module it restores there runs the reverted build, which it
|
||||
// starts only once it has said `restored` (OrderEngineLast);
|
||||
// - the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a source revision at least
|
||||
// the trunk's head, the highest `src` of the trunk branch's module values (OrderTakenIn, A-260).
|
||||
|
||||
// takenSubject is the indexer's event take-in.taken on its own stream, as the manifest's consumes names it.
|
||||
var takenSubject = process.ModuleStreamSubject(buckets.IndexerModule, "take-in.taken")
|
||||
|
||||
// takenSources are the newest source revisions a take-in.taken named, by watched branch, as heard (hearBuildsAndReleases).
|
||||
type takenSources struct {
|
||||
mu sync.Mutex
|
||||
own map[string]int64
|
||||
}
|
||||
|
||||
// put takes one take-in.taken in. A hand drop, and a taken with no source revision (one said before G), take nothing in.
|
||||
func (t *takenSources) put(data []byte) error {
|
||||
var e meshcatalog.TakeInTaken
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return fmt.Errorf("a take-in.taken that is none: %w", err)
|
||||
}
|
||||
if e.Dropped || e.Source == 0 || e.Branch == "" {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.own == nil {
|
||||
t.own = map[string]int64{}
|
||||
}
|
||||
t.own[e.Branch] = max(t.own[e.Branch], e.Source)
|
||||
return nil
|
||||
}
|
||||
|
||||
// of is the newest source revision of branch taken in; 0 before one is heard.
|
||||
func (t *takenSources) of(branch string) int64 {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.own[branch]
|
||||
}
|
||||
|
||||
// orderEdge is one order edge held on module's want on node: part's version fp must run on partNode (OrderBackup, OrderEngineLast), or the
|
||||
// trunk's source revision source be taken in (OrderTakenIn).
|
||||
type orderEdge struct {
|
||||
node, module, part, partNode, fp, entry, order string
|
||||
source int64
|
||||
}
|
||||
|
||||
func (e orderEdge) key() string { return acting.OrderKey(e.node, e.module, e.part) }
|
||||
|
||||
func (e orderEdge) acting() assign.ActingEdge {
|
||||
return assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace}
|
||||
}
|
||||
|
||||
// orderRuns says an order edge is met: its part runs the version, or the source revision is taken in.
|
||||
func (p *passer) orderRuns(n assign.Nodes, e orderEdge) bool {
|
||||
if e.order == acting.OrderTakenIn {
|
||||
return p.w.taken.of(p.w.trunkName()) >= e.source
|
||||
}
|
||||
return n.Runs(e.acting())
|
||||
}
|
||||
|
||||
// seatHolders are the holders of each seat as the assignments compose them, by seat name.
|
||||
func (p *passer) holdersBySeat() map[string][]mma.SeatHoldersHoldersItem {
|
||||
out := map[string][]mma.SeatHoldersHoldersItem{}
|
||||
for _, sh := range assignment.SeatHoldersOf(p.w.assignments()) {
|
||||
out[sh.Seat] = append(out[sh.Seat], sh.Holders...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// orderEdgesOf computes §298's edges on the wants of module m on each of its nodes, into p.orders. wanted is m's wants by node; trunk the
|
||||
// trunk's module values. A node that has no running build heard, runs m at its trunk version already, or is wanted that version already
|
||||
// has none: nothing moves. It answers, by node, whether an edge still stands that cannot be recorded as a wait because the catalog entry
|
||||
// it reads is not answered yet (asked, and its answer wakes a pass): the want is held meanwhile, never written across a form change unread.
|
||||
func (p *passer) orderEdgesOf(ctx context.Context, m assign.Module, wanted map[string]mwc.Want, trunk map[string]meshcatalog.ModulesByModuleValue) map[string]bool {
|
||||
unread := map[string]bool{}
|
||||
p.orderVisited[m.Name] = true
|
||||
if m.FP == "" || p.graph == nil || p.entries == nil {
|
||||
return unread
|
||||
}
|
||||
var holders map[string][]mma.SeatHoldersHoldersItem
|
||||
for _, node := range m.Nodes {
|
||||
if w, ok := wanted[node]; ok && !w.Removed && w.Fp == m.FP {
|
||||
continue // the version is wanted already: nothing moves, so nothing waits
|
||||
}
|
||||
run, heard := p.w.runningOf(node, m.Name)
|
||||
if !heard || run.Stopped || run.Fingerprint == "" || run.Fingerprint == m.FP {
|
||||
continue
|
||||
}
|
||||
if holders == nil {
|
||||
holders = p.holdersBySeat()
|
||||
}
|
||||
held := func(seatName string) bool {
|
||||
for _, h := range holders[seatName] {
|
||||
if h.Module == m.Name && h.Node == node {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
add := func(e orderEdge) {
|
||||
e.node, e.module, e.entry = node, m.Name, m.FP
|
||||
p.orders[e.key()] = e
|
||||
}
|
||||
// 1. a kept data's form changes: the restore point needs the new node-backup holder and bus module.
|
||||
changes, known := p.formChange(ctx, m.Name, run.Fingerprint, m.FP)
|
||||
if !known {
|
||||
unread[node] = true
|
||||
}
|
||||
if len(changes) > 0 {
|
||||
for seatName, onNode := range map[string]bool{catalogue.BackupSeat: true, meshbroker.Seat: false} {
|
||||
for _, h := range holders[seatName] {
|
||||
if h.Module == m.Name || (onNode && h.Node != node) || trunk[h.Module].Fp2 == "" {
|
||||
continue
|
||||
}
|
||||
add(orderEdge{part: h.Module, partNode: h.Node, fp: trunk[h.Module].Fp2, order: acting.OrderBackup})
|
||||
}
|
||||
}
|
||||
}
|
||||
if !held(nodeengine.Seat) && !held(mma.Seat) && !held(meshdelivery.Seat) {
|
||||
continue
|
||||
}
|
||||
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, m.Name, run.Fingerprint, m.FP)
|
||||
if err != nil {
|
||||
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", m.Name, node, m.FP, err)
|
||||
continue
|
||||
}
|
||||
if !reverted {
|
||||
continue
|
||||
}
|
||||
// 2. at a revert the node-engine goes back last on its node: after every module that is set back there across a form change runs.
|
||||
if held(nodeengine.Seat) {
|
||||
for module, byNode := range p.w.wantsOf() {
|
||||
// What it is set back to is its trunk version, which the rules write as its want in this pass.
|
||||
target := trunk[module].Fp2
|
||||
if w, ok := byNode[node]; module == m.Name || !ok || w.Removed || target == "" {
|
||||
continue
|
||||
}
|
||||
xrun, heard := p.w.runningOf(node, module)
|
||||
if !heard || xrun.Stopped || xrun.Fingerprint == "" || xrun.Fingerprint == target {
|
||||
continue
|
||||
}
|
||||
if back, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, xrun.Fingerprint, target); err != nil || !back {
|
||||
continue
|
||||
}
|
||||
xchanges, known := p.formChange(ctx, module, xrun.Fingerprint, target)
|
||||
if !known {
|
||||
unread[node] = true
|
||||
}
|
||||
if len(xchanges) > 0 {
|
||||
add(orderEdge{part: module, partNode: node, fp: target, order: acting.OrderEngineLast})
|
||||
}
|
||||
}
|
||||
}
|
||||
// 3. the reverted assigner and gate wait for the indexer to take in the trunk's head.
|
||||
if held(mma.Seat) || held(meshdelivery.Seat) {
|
||||
var head int64
|
||||
for _, v := range p.w.trunkOf(p.w.trunkName()) {
|
||||
head = max(head, v.Src)
|
||||
}
|
||||
if head > 0 {
|
||||
add(orderEdge{part: buckets.IndexerModule, fp: strconv.FormatInt(head, 10), source: head, order: acting.OrderTakenIn})
|
||||
}
|
||||
}
|
||||
}
|
||||
return unread
|
||||
}
|
||||
|
||||
// orderActing is the order edges of module on node that still hold its want, as acting edges: those not met, and not released by hand. A
|
||||
// node whose catalog entry is not read yet holds its want too, on a stand-in edge no part runs.
|
||||
func (p *passer) orderActing(module string, released map[string]bool, unread map[string]bool, n func() assign.Nodes) func(node string) []assign.ActingEdge {
|
||||
return func(node string) []assign.ActingEdge {
|
||||
var out []assign.ActingEdge
|
||||
nodes := n()
|
||||
for _, e := range p.orders {
|
||||
if e.module == module && e.node == node && !released[e.key()+"@"+e.fp] && !p.orderRuns(nodes, e) {
|
||||
out = append(out, assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace})
|
||||
}
|
||||
}
|
||||
if unread[node] {
|
||||
out = append(out, assign.ActingEdge{Node: node, Part: "the catalog entry", Step: seatdef.StepPlace})
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// orderWaits keeps the order waits after the pass's wants, as actingWaits keeps the acting waits: it starts the wait of each edge not met,
|
||||
// ends the wait of one that ran, was replaced or was released, and says each.
|
||||
func (p *passer) orderWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]mwc.Want) {
|
||||
if p.kvs[buckets.Waits()] == nil {
|
||||
return
|
||||
}
|
||||
now := p.clock()
|
||||
kept := p.w.read(buckets.Waits())
|
||||
trunk := p.w.trunk()
|
||||
if p.orderSaid == nil {
|
||||
p.orderSaid = map[string]bool{}
|
||||
}
|
||||
standing := map[string]bool{}
|
||||
for key, e := range p.orders {
|
||||
if w, ok := wanted[e.module][e.node]; ok && !w.Removed && w.Fp == e.entry {
|
||||
continue
|
||||
}
|
||||
if p.orderRuns(n, e) {
|
||||
continue
|
||||
}
|
||||
standing[key] = true
|
||||
if raw := kept[key]; raw != nil {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Entry == e.entry && w.Fp == e.fp {
|
||||
if !w.Released && !p.orderSaid[key] {
|
||||
// A wait still held after a restart is said again, as wait-held (A-260).
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry+".again"); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said again, and is said at the next pass: %v", e.module, e.node, e.part, err)
|
||||
} else {
|
||||
p.orderSaid[key] = true
|
||||
}
|
||||
}
|
||||
p.actingReleaseSaid(ctx, key, w)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !n.Online[e.node] {
|
||||
continue // a node away holds only itself, and its clock has not started (§268)
|
||||
}
|
||||
w := acting.Wait{Step: seatdef.StepPlace, Node: e.node, Module: e.module, Part: e.part, PartNode: e.partNode, Fp: e.fp, Entry: e.entry, Order: e.order,
|
||||
Since: now, ClockFrom: now, Bound: acting.BoundOf(seat.Edge{Step: seatdef.StepPlace}).String()}
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", e.module, e.node, e.part, err)
|
||||
continue
|
||||
}
|
||||
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
p.orderSaid[key] = true
|
||||
}
|
||||
for key, raw := range kept {
|
||||
if !acting.IsOrder(key) || standing[key] {
|
||||
continue
|
||||
}
|
||||
w, err := acting.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, there := trunk[w.Module]; there && !p.orderVisited[w.Module] {
|
||||
continue // the module was not looked at in this pass (no pipeline passed for it yet): its wait stands as kept
|
||||
}
|
||||
how := acting.HowRan
|
||||
switch e, there := p.orders[key]; {
|
||||
case w.Released:
|
||||
how = acting.HowReleased
|
||||
case there && e.fp != w.Fp, !there && wanted[w.Module][w.Node].Fp != w.Entry:
|
||||
how = acting.HowReplaced
|
||||
}
|
||||
if !w.EndSaid {
|
||||
body := acting.Body(w, assignment.Module())
|
||||
body.How = how
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
|
||||
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.remove(ctx, buckets.Waits(), key)
|
||||
delete(p.orderSaid, key)
|
||||
}
|
||||
}
|
||||
@@ -1,351 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
|
||||
func orderFP(c string) string {
|
||||
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
|
||||
}
|
||||
|
||||
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
|
||||
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
|
||||
type orderMod struct {
|
||||
name string
|
||||
run, trunk string
|
||||
forms map[string]int
|
||||
holds []string
|
||||
src int64
|
||||
}
|
||||
|
||||
type orderFixture struct {
|
||||
t *testing.T
|
||||
p *passer
|
||||
wants *memKV
|
||||
events []string
|
||||
bodies []meshwants.Wait
|
||||
answers chan struct{}
|
||||
}
|
||||
|
||||
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
|
||||
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
|
||||
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
|
||||
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
|
||||
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p := f.p
|
||||
p.logf = t.Logf
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
f.events = append(f.events, event)
|
||||
f.bodies = append(f.bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
pl := mma.Assignments{Node: "ace"}
|
||||
forms := map[string]int{} // by fingerprint: each module's own
|
||||
modules := []string{}
|
||||
for _, m := range mods {
|
||||
modules = append(modules, m.name)
|
||||
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
|
||||
for _, seat := range m.holds {
|
||||
scope := "node"
|
||||
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
|
||||
scope = "mesh"
|
||||
}
|
||||
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
|
||||
}
|
||||
for fp, form := range m.forms {
|
||||
forms[fp] = form
|
||||
}
|
||||
mdigest := fmt.Sprintf("%064x", len(modules))
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, Src: m.src})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
endDone(t, p.w, m.name, m.trunk, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
f.runs(m.name, m.run)
|
||||
wkey, _ := mwc.WantKey("ace", m.name)
|
||||
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
|
||||
p.w.put(buckets.Wants(), wkey, old)
|
||||
f.wants.values[wkey] = old
|
||||
}
|
||||
composes(t, p.w, "ace", digest)
|
||||
values, err := mmac.AssignmentsValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(plc.AssignmentsBucket(), key, raw)
|
||||
}
|
||||
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
|
||||
objects := memObjects{}
|
||||
for i, at := range history {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, module := range modules {
|
||||
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
|
||||
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
|
||||
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
|
||||
}
|
||||
placed := mma.Assignments{Node: "ace"}
|
||||
for _, module := range modules {
|
||||
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
|
||||
}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]mma.Assignments{"ace": placed}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
|
||||
if form := forms[in.Fingerprint]; form > 0 {
|
||||
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
|
||||
return f
|
||||
}
|
||||
|
||||
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
|
||||
func (f *orderFixture) runs(module, fp string) {
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
f.p.w.put(buckets.Have, "ace."+module, have)
|
||||
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
|
||||
}
|
||||
|
||||
// pass makes a pass, and after it every catalog entry it asked is answered.
|
||||
func (f *orderFixture) pass() {
|
||||
f.t.Helper()
|
||||
f.p.wants(context.Background())
|
||||
asked := 0
|
||||
for _, row := range f.p.entries.Rows() {
|
||||
if row.State == meshcatalogclient.EntryAsked {
|
||||
asked++
|
||||
}
|
||||
}
|
||||
for range asked {
|
||||
select {
|
||||
case <-f.answers:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
f.t.Fatal("an entry asked was not answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wantAt is the fingerprint module is wanted at on ace.
|
||||
func (f *orderFixture) wantAt(module string) string {
|
||||
f.t.Helper()
|
||||
key, _ := mwc.WantKey("ace", module)
|
||||
w, err := mwc.DecodeWant(f.wants.values[key])
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
|
||||
func (f *orderFixture) waits() []string {
|
||||
var out []string
|
||||
for key := range f.p.w.read(buckets.Waits()) {
|
||||
if acting.IsOrder(key) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Design 47 G §298 first edge, A-258, A-259: a module whose kept data's form changes is not moved until the new version of the node-backup
|
||||
// holder and of the holder of the seat mesh-broker run; the wait is kept under `order.` in the state waits, said as wait-held, and ends
|
||||
// ran (wait-ended) when the holders run; a module whose kept data keeps its form is moved at once.
|
||||
func TestAModuleWhoseKeptDataChangesFormWaitsForTheNewBackupHolderAndTheBusModule(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "plain", run: orderFP("c"), trunk: orderFP("d"), forms: map[string]int{orderFP("c"): 1, orderFP("d"): 1}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
|
||||
})
|
||||
f.pass() // the entries are asked: no want crosses a form change unread
|
||||
if got := f.wantAt("web"); got != orderFP("a") {
|
||||
t.Fatalf("web is wanted at %s before the entries are read", got)
|
||||
}
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("a") {
|
||||
t.Fatalf("web moved across a form change at %s while the new node-backup holder and bus module do not run", got)
|
||||
}
|
||||
if got := f.wantAt("plain"); got != orderFP("d") {
|
||||
t.Errorf("a module whose kept data keeps its form is wanted at %s, not moved", got)
|
||||
}
|
||||
if len(f.waits()) != 2 || len(f.events) != 2 || f.events[0] != meshwants.EventWaitHeld || f.bodies[0].Step != seatdef.StepPlace || f.bodies[0].Module != "web" {
|
||||
t.Fatalf("waits %v events %v bodies %+v: want two wait-held in the step place, `order.ace.web.backup` and `order.ace.web.bus`", f.waits(), f.events, f.bodies)
|
||||
}
|
||||
if _, err := acting.Decode(f.p.w.read(buckets.Waits())[acting.OrderKey("ace", "web", "backup")]); err != nil {
|
||||
t.Fatalf("the wait is not kept under its key: %v", err)
|
||||
}
|
||||
f.pass()
|
||||
if len(f.events) != 2 {
|
||||
t.Fatalf("a pass that changed nothing said %v", f.events)
|
||||
}
|
||||
f.runs("backup", orderFP("f"))
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("a") || len(f.events) != 3 || f.events[2] != meshwants.EventWaitEnded || f.bodies[2].How != acting.HowRan {
|
||||
t.Fatalf("with the bus module still old: web at %s, events %v", got, f.events)
|
||||
}
|
||||
f.runs("bus", orderFP("h"))
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 4 {
|
||||
t.Fatalf("with both running: web at %s, waits %v, events %v", got, f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
|
||||
// §298 first edge, the hand: assigner_waits reads an order wait and assigner_release with how `now` lets the module move.
|
||||
func TestAnOrderWaitIsReadAndReleasedByTheExistingTools(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
})
|
||||
f.pass()
|
||||
f.pass()
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: f.p.kvs[buckets.Waits()].(*memKV), buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
got, err := s.doWaits(map[string]any{"module": "web"})
|
||||
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).Key != "order.ace.web.backup" {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "web", "how": "now", "why": "the restore point is taken by hand"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
|
||||
t.Fatalf("%v %v", err, acts)
|
||||
}
|
||||
f.p.w.put(buckets.Waits(), "order.ace.web.backup", f.p.kvs[buckets.Waits()].(*memKV).values["order.ace.web.backup"])
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || f.bodies[len(f.bodies)-1].How != acting.HowReleased {
|
||||
t.Fatalf("after the release: web at %s, last %+v", got, f.bodies[len(f.bodies)-1])
|
||||
}
|
||||
}
|
||||
|
||||
// §298 second edge, A-259: at a revert the node-engine goes back last on its node, after every module that is set back there across a
|
||||
// form change runs the reverted build.
|
||||
func TestAtARevertTheNodeEngineGoesBackLast(t *testing.T) {
|
||||
old, now := map[string]string{"web": orderFP("a"), "engine": orderFP("c")}, map[string]string{"web": orderFP("b"), "engine": orderFP("d")}
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("b"), trunk: orderFP("a"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "engine", run: orderFP("d"), trunk: orderFP("c"), holds: []string{nodeengine.Seat}},
|
||||
}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("web") != orderFP("a") {
|
||||
t.Fatalf("web is wanted at %s: the module set back is not held", f.wantAt("web"))
|
||||
}
|
||||
if got := f.wantAt("engine"); got != orderFP("d") {
|
||||
t.Fatalf("the node-engine is set back to %s before web runs its reverted build", got)
|
||||
}
|
||||
if w := f.waits(); len(w) != 1 || w[0] != "order.ace.engine.web" || f.bodies[len(f.bodies)-1].Module != "engine" {
|
||||
t.Fatalf("waits %v bodies %+v", w, f.bodies)
|
||||
}
|
||||
f.runs("web", orderFP("a"))
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("engine"); got != orderFP("c") || len(f.waits()) != 0 {
|
||||
t.Fatalf("with web back: the node-engine is wanted at %s, waits %v", got, f.waits())
|
||||
}
|
||||
}
|
||||
|
||||
// §298 third edge, A-259, A-260: the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a
|
||||
// source revision at least the trunk's head (the highest src of the trunk's module values).
|
||||
func TestTheRevertedAssignerAndGateWaitForTheIndexerToTakeInTheTrunksHead(t *testing.T) {
|
||||
old, now := map[string]string{"asg": orderFP("a"), "gate": orderFP("c"), "idx": orderFP("e")}, map[string]string{"asg": orderFP("b"), "gate": orderFP("d"), "idx": orderFP("f")}
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 6},
|
||||
{name: "gate", run: orderFP("d"), trunk: orderFP("c"), holds: []string{meshdelivery.Seat}, src: 7},
|
||||
{name: "idx", run: orderFP("e"), trunk: orderFP("e"), src: 7},
|
||||
}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("b") || f.wantAt("gate") != orderFP("d") {
|
||||
t.Fatalf("the reverted assigner (%s) and gate (%s) are wanted before the head is taken in", f.wantAt("asg"), f.wantAt("gate"))
|
||||
}
|
||||
if len(f.waits()) != 2 || f.bodies[0].Part != "mesh-delivery-indexer" {
|
||||
t.Fatalf("waits %v bodies %+v", f.waits(), f.bodies)
|
||||
}
|
||||
taken := func(branch string, source int64) {
|
||||
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: branch, Source: source})
|
||||
if err := f.p.w.taken.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
taken("beta", 9) // another branch, and
|
||||
taken(trunkBranch, 6)
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("b") {
|
||||
t.Fatal("a source revision before the trunk's head was taken in, and the assigner was wanted")
|
||||
}
|
||||
taken(trunkBranch, 7)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("a") || f.wantAt("gate") != orderFP("c") || len(f.waits()) != 0 {
|
||||
t.Fatalf("after the head is taken in: asg %s gate %s waits %v", f.wantAt("asg"), f.wantAt("gate"), f.waits())
|
||||
}
|
||||
}
|
||||
|
||||
// A-260: an order wait still held after a restart is said again as wait-held, by a pass that remembers nothing, and ends at the next
|
||||
// take-in.taken that names the trunk's head.
|
||||
func TestAnOrderWaitHeldAfterARestartIsSaidAgain(t *testing.T) {
|
||||
old, now := map[string]string{"asg": orderFP("a")}, map[string]string{"asg": orderFP("b")}
|
||||
f := newOrderFixture(t, []orderMod{{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 7}}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if len(f.events) != 1 {
|
||||
t.Fatalf("events %v", f.events)
|
||||
}
|
||||
f.p.orderSaid = nil // a process started again
|
||||
f.pass()
|
||||
if len(f.events) != 2 || f.events[1] != meshwants.EventWaitHeld || len(f.waits()) != 1 {
|
||||
t.Fatalf("events %v waits %v", f.events, f.waits())
|
||||
}
|
||||
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: trunkBranch, Source: 8})
|
||||
if err := f.p.w.taken.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("a") || len(f.waits()) != 0 || f.events[len(f.events)-1] != meshwants.EventWaitEnded {
|
||||
t.Fatalf("asg at %s, waits %v, events %v", f.wantAt("asg"), f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
@@ -76,9 +76,6 @@ type outsideActs struct {
|
||||
cleared map[string]bool
|
||||
// sigs is each check's last signature, by node.module.
|
||||
sigs map[string]string
|
||||
// own is the machines whose own network is open; ownRead says the read at the start was answered.
|
||||
own map[string]bool
|
||||
ownRead bool
|
||||
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
|
||||
causes map[string][]string
|
||||
// started says the first pass judged every wait again.
|
||||
@@ -93,7 +90,7 @@ func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func()
|
||||
settle = outside.DefaultSettle
|
||||
}
|
||||
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
|
||||
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, own: map[string]bool{}, causes: map[string][]string{}}
|
||||
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, causes: map[string][]string{}}
|
||||
}
|
||||
|
||||
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
|
||||
@@ -135,31 +132,8 @@ func settingFile(path string) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// readStart reads the open conditions of every machine once at the start, for the machines whose own network is held.
|
||||
func (o *outsideActs) readStart() {
|
||||
go func() {
|
||||
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine"})
|
||||
if err != nil {
|
||||
o.logf("the open conditions are not read at the start (%v): no machine is held for its own network until its event is heard", err)
|
||||
return
|
||||
}
|
||||
conds, err := outside.ParseConditions(answer)
|
||||
if err != nil {
|
||||
o.logf("%v", err)
|
||||
return
|
||||
}
|
||||
o.mu.Lock()
|
||||
for m := range outside.OwnNetwork(conds) {
|
||||
o.own[m] = true
|
||||
}
|
||||
o.ownRead = true
|
||||
o.mu.Unlock()
|
||||
o.wake()
|
||||
}()
|
||||
}
|
||||
|
||||
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again and keeps the
|
||||
// machines whose own network is open; nothing is kept of the condition.
|
||||
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again; nothing is kept
|
||||
// of the condition (the network hold ends, design 47 H §750, H-A-55).
|
||||
func (o *outsideActs) Heard(subject string, body []byte) {
|
||||
var e mcseat.ConditionEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
|
||||
@@ -168,9 +142,6 @@ func (o *outsideActs) Heard(subject string, body []byte) {
|
||||
}
|
||||
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
|
||||
o.mu.Lock()
|
||||
if p := strings.Split(e.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" {
|
||||
o.own[p[1]] = !cleared
|
||||
}
|
||||
if cleared {
|
||||
for _, node := range o.causes[e.Key] {
|
||||
o.dirty[node] = true
|
||||
@@ -186,16 +157,6 @@ func (o *outsideActs) Heard(subject string, body []byte) {
|
||||
o.wake()
|
||||
}
|
||||
|
||||
// Held says whether machine's own network is open: no want moves to it.
|
||||
func (o *outsideActs) Held(machine string) bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.own[machine]
|
||||
}
|
||||
|
||||
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
|
||||
// which it starts when none is in flight.
|
||||
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
|
||||
|
||||
@@ -237,29 +237,12 @@ func TestTheWaitOnAnOutsideCause(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestAMachineWhoseOwnNetworkIsOpenGetsNoWant(t *testing.T) {
|
||||
m := assign.Module{Name: "nats", FP: fpTest, Built: true, Nodes: []string{"ace"}, MachineHeld: func(node string) bool { return node == "ace" }}
|
||||
ok, rule, why := assign.Holds(m, "ace", assign.Nodes{Online: map[string]bool{"ace": true}})
|
||||
if ok || rule == 0 || !strings.Contains(why, "machine.ace.network") {
|
||||
t.Errorf("holds = %v %d %q", ok, rule, why)
|
||||
}
|
||||
o := newOutsideActs(nil, func() {}, func(string, ...any) {}, 0, 0)
|
||||
o.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
|
||||
if !o.Held("ace") || o.Held("g14") {
|
||||
t.Error("the own network condition's event does not hold the machine")
|
||||
}
|
||||
o.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
|
||||
if o.Held("ace") {
|
||||
t.Error("the machine is held after its condition cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "ace")
|
||||
r.review(false, nil)
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
@@ -296,38 +279,4 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
|
||||
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
|
||||
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
|
||||
}
|
||||
// A-246: a dependent's delivery wait is listed beside them, and released by its module and step delivery, its hand act first.
|
||||
dw, _ := json.Marshal(deliveryWaiting{Wait: meshwants.Wait{Step: meshwants.WaitStepDelivery, Module: "app", Part: "db", On: "db@1.1.0", Machines: []string{"g14"}}})
|
||||
s.st.put(buckets.DeliveryWaitsState, "app", dw)
|
||||
got, err = s.doWaits(map[string]any{"node": "g14"})
|
||||
if d := got.(map[string]any)["delivery"].([]deliveryWaiting); err != nil || len(d) != 1 || d[0].Module != "app" {
|
||||
t.Errorf("the delivery waits listed = %+v %v", got, err)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"module": "app", "step": "build", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a release of another step is taken")
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"module": "web", "step": "delivery", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a release of a module waiting for no delivery is taken")
|
||||
}
|
||||
before := len(acts)
|
||||
if _, err := s.doRelease(map[string]any{"module": "app", "step": "delivery", "why": "db is fine as it runs"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _, _, _ = s.st.get(buckets.DeliveryWaitsState, "app")
|
||||
var released deliveryWaiting
|
||||
if json.Unmarshal(raw, &released) != nil || !released.Released || !strings.Contains(released.By, "db is fine") || len(acts) != before+1 {
|
||||
t.Errorf("the delivery wait after its release = %+v, hand acts %v", released, acts[before:])
|
||||
}
|
||||
// A-250: assigner_delivered_again records its hand act, writes the ask and answers asked; assigner_waits lists it.
|
||||
if _, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0"}, "op"); err == nil {
|
||||
t.Error("an ask with no why is taken")
|
||||
}
|
||||
ans, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0", "why": "the catalog missed it"}, "op")
|
||||
if err != nil || ans.(map[string]any)["asked"] != "db" || len(acts) != before+2 {
|
||||
t.Fatalf("assigner_delivered_again = %+v %v", ans, err)
|
||||
}
|
||||
got, _ = s.doWaits(map[string]any{})
|
||||
if a := got.(map[string]any)["delivered-again"].([]againAsk); len(a) != 1 || a[0].Version != "1.1.0" || a[0].By != "op" {
|
||||
t.Errorf("the asks listed = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
@@ -10,11 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/outside"
|
||||
)
|
||||
@@ -37,13 +33,6 @@ type waitSaid struct {
|
||||
BoundEnds string `json:"bound-ends,omitempty"`
|
||||
}
|
||||
|
||||
// actingSaid is one wait for an acting part's new version as assigner_waits answers it (actingwaits.go).
|
||||
type actingSaid struct {
|
||||
Key string `json:"key"`
|
||||
acting.Wait
|
||||
BoundEnds string `json:"bound-ends,omitempty"`
|
||||
}
|
||||
|
||||
// doWaits is the tool assigner_waits.
|
||||
func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(waitsState)
|
||||
@@ -57,16 +46,6 @@ func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if acting.Is(key) || acting.IsOrder(key) {
|
||||
if a, err := acting.Decode(raw); err == nil && (arg(args, "node") == "" || a.Node == arg(args, "node")) && (arg(args, "module") == "" || a.Module == arg(args, "module")) {
|
||||
said := actingSaid{Key: key, Wait: a}
|
||||
if d, err := time.ParseDuration(a.Bound); err == nil && !a.Released {
|
||||
said.BoundEnds = a.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w, err := outside.Decode(raw)
|
||||
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
|
||||
continue
|
||||
@@ -79,120 +58,13 @@ func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
// And each dependent's wait for its dependency's delivery (design 47 G §376, A-246), from the state delivery-waits.
|
||||
delivery := []deliveryWaiting{}
|
||||
dkeys, err := s.st.keys(buckets.DeliveryWaitsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the delivery waits cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(dkeys)
|
||||
for _, key := range dkeys {
|
||||
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var w deliveryWaiting
|
||||
if json.Unmarshal(raw, &w) != nil || (arg(args, "module") != "" && w.Module != arg(args, "module")) ||
|
||||
(arg(args, "node") != "" && !slices.Contains(w.Machines, arg(args, "node"))) {
|
||||
continue
|
||||
}
|
||||
delivery = append(delivery, w)
|
||||
}
|
||||
// And each ask of assigner_delivered_again not yet holding, with why (A-250).
|
||||
again := []againAsk{}
|
||||
akeys, err := s.st.keys(buckets.DeliveredAgainState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the asks to say a version delivered again cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(akeys)
|
||||
for _, key := range akeys {
|
||||
raw, _, found, err := s.st.get(buckets.DeliveredAgainState, key)
|
||||
var a againAsk
|
||||
if err != nil || !found || json.Unmarshal(raw, &a) != nil || (arg(args, "module") != "" && a.Module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
again = append(again, a)
|
||||
}
|
||||
return map[string]any{"waits": out, "delivery": delivery, "delivered-again": again}, nil
|
||||
}
|
||||
|
||||
// toolDeliveredAgain is the tool that asks version-delivered said again for one version (design 47 G §376, A-250).
|
||||
const toolDeliveredAgain = "assigner_delivered_again"
|
||||
|
||||
// doDeliveredAgain is assigner_delivered_again: the hand act is recorded first, then the ask is written into the state
|
||||
// delivered-again, whose watch the assigner's process hears at once; it answers that it is asked.
|
||||
func (s *putTools) doDeliveredAgain(args map[string]any, by string) (any, error) {
|
||||
module, version, why := arg(args, "module"), arg(args, "version"), arg(args, "why")
|
||||
if module == "" || version == "" || why == "" {
|
||||
return nil, errors.New("assigner_delivered_again names the module, its version and why: it is kept in the hand-act log")
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_delivered_again %s %s", module, version), Why: why,
|
||||
Cause: "version-delivered", Verb: "mesh-wants.assigner_delivered_again"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err)
|
||||
}
|
||||
data, err := json.Marshal(againAsk{Module: module, Version: version, By: by, Why: why})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(buckets.DeliveredAgainState, keys.EscapeName(module), data); err != nil {
|
||||
return nil, fmt.Errorf("the ask of %s %s is not written: %w", module, version, err)
|
||||
}
|
||||
return map[string]any{"asked": module, "version": version, "hand-act": taken.Log,
|
||||
"follow": "the assigner says version-delivered again once every present machine placing it runs it healthy; assigner_waits says why it does not yet"}, nil
|
||||
}
|
||||
|
||||
// deliveredAgainTool is assigner_delivered_again as the runtime serves it.
|
||||
func (s *putTools) deliveredAgainTool() stdio.Tool {
|
||||
return stdio.Tool{Name: toolDeliveredAgain, Description: "Say version-delivered again for one version of a module, for the operator and the healers " +
|
||||
"(design 47 G §376): written in the hand-act log first, then asked of the assigner, which says it once every present machine placing " +
|
||||
"the module runs that version healthy; until then assigner_waits lists the ask with why it does not hold. Answers that it is asked.",
|
||||
Input: schema(map[string]string{"module": "the module", "version": "its MAJOR.MINOR.PATCH", "why": "why, kept in the hand-act log"},
|
||||
[]string{"module", "version", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doDeliveredAgain(a, stdio.CallerOf(a)) }}
|
||||
}
|
||||
|
||||
// doReleaseDelivery is assigner_release with step delivery: the hand act is recorded first, then the release is written on the
|
||||
// module's delivery wait, which the assigner's process ends released at its next pass, writing the module's want.
|
||||
func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error) {
|
||||
module, why := arg(args, "module"), arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept in the hand-act log")
|
||||
}
|
||||
key := keys.EscapeName(module)
|
||||
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the delivery wait of %s is not read: %w", module, err)
|
||||
}
|
||||
var w deliveryWaiting
|
||||
if !found || json.Unmarshal(raw, &w) != nil || w.Released {
|
||||
return nil, fmt.Errorf("%s waits for no delivery: assigner_waits lists the delivery waits", module)
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s step delivery (waits on %s)", module, w.On), Why: why,
|
||||
Cause: "delivery-wait", Verb: "mesh-wants.assigner_release"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
|
||||
}
|
||||
w.Released, w.By = true, by+": "+why
|
||||
data, err := json.Marshal(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(buckets.DeliveryWaitsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s's delivery wait is not written: %w", module, err)
|
||||
}
|
||||
return map[string]any{"released": module, "step": meshwants.WaitStepDelivery, "by": by, "hand-act": taken.Log}, nil
|
||||
return map[string]any{"waits": out}, nil
|
||||
}
|
||||
|
||||
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
|
||||
// module, which the assigner's process takes at its next pass.
|
||||
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
|
||||
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
|
||||
if how == acting.HowNow {
|
||||
return s.doReleaseActing(node, module, why, by)
|
||||
}
|
||||
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
|
||||
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
|
||||
}
|
||||
@@ -238,61 +110,9 @@ func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
|
||||
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// doReleaseActing is assigner_release with how `now`: the entry of module on node moves with its acting parts' running versions. The
|
||||
// hand act is recorded first; the release is written on each standing wait of the entry, and the process says wait-ended at its next pass.
|
||||
func (s *putTools) doReleaseActing(node, module, why, by string) (any, error) {
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept in the hand-act log")
|
||||
}
|
||||
keys, err := s.st.keys(waitsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
|
||||
}
|
||||
type one struct {
|
||||
key string
|
||||
w acting.Wait
|
||||
}
|
||||
var held []one
|
||||
prefix, orderPrefix := acting.Prefix+node+"."+module+".", acting.OrderPrefix+node+"."+module+"."
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, orderPrefix) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(waitsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if w, err := acting.Decode(raw); err == nil && !w.Released {
|
||||
held = append(held, one{k, w})
|
||||
}
|
||||
}
|
||||
if len(held) == 0 {
|
||||
return nil, fmt.Errorf("%s holds no wait of %s for an acting part: assigner_waits lists them", node, module)
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, acting.HowNow), Why: why,
|
||||
Cause: "acting-wait", Verb: "mesh-wants.assigner_release"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
|
||||
}
|
||||
released := []string{}
|
||||
for _, h := range held {
|
||||
h.w.Released, h.w.By, h.w.Why = true, by+": "+why, why
|
||||
data, err := acting.Encode(h.w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(waitsState, h.key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
released = append(released, h.key)
|
||||
}
|
||||
return map[string]any{"released": released, "how": acting.HowNow, "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// waitsTool is assigner_waits as the runtime serves it.
|
||||
func (s *putTools) waitsTool() stdio.Tool {
|
||||
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: an entry waiting for an acting part's new version in composing or placing (§266, §268), with its part, version, step, clock-from and bound; and a moved module's failing health check on a " +
|
||||
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: a moved module's failing health check on a " +
|
||||
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
|
||||
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
|
||||
"machine or module. Read-only.",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,539 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
|
||||
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat
|
||||
// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its
|
||||
// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state`
|
||||
// section `pins` (4 MiB), one key `trunk.<n>` per assignment graph revision whose value is the list of the records pinning
|
||||
// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys:
|
||||
// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision
|
||||
// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names
|
||||
// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up
|
||||
// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being
|
||||
// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state.
|
||||
|
||||
const (
|
||||
pinsState = "pins"
|
||||
pinsTool = "assigner_pins"
|
||||
unpinTool = "assigner_unpin"
|
||||
pinsMadeKey = "made"
|
||||
pinsStatedKey = "pins-stated"
|
||||
pinsCapBytes = 4 << 20
|
||||
pinMissingKind = "pin-missing"
|
||||
)
|
||||
|
||||
// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held
|
||||
// when a record pinned it.
|
||||
type PinsValue struct {
|
||||
Records []string `json:"records"`
|
||||
Missing bool `json:"missing,omitempty"`
|
||||
}
|
||||
|
||||
// pinKey is the key of an assignment graph revision, its object's own name (A-169).
|
||||
func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) }
|
||||
|
||||
// pinRevision is the revision a key of the state names, and whether it names one.
|
||||
func pinRevision(key string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(key, "trunk.")
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
r, err := strconv.ParseInt(n, 10, 64)
|
||||
return r, err == nil && r > 0
|
||||
}
|
||||
|
||||
// pinObjects is the object bucket of the revisions as the pins remove from it.
|
||||
type pinObjects interface {
|
||||
assigngraph.Objects
|
||||
Delete(ctx context.Context, name string) error
|
||||
}
|
||||
|
||||
// pinKeeper keeps the pins and removes what nothing pins.
|
||||
type pinKeeper struct {
|
||||
ctx context.Context
|
||||
st endStore
|
||||
objects pinObjects
|
||||
// inUse says a want or a running build names the module's fingerprint (§179).
|
||||
inUse func(module, fp string) bool
|
||||
findings assign.Findings
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
// caught is whether the last event applied left none of the consumer's messages pending.
|
||||
caught bool
|
||||
}
|
||||
|
||||
// edit applies change to the value at key by compare-and-set, again while another write comes between.
|
||||
func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error {
|
||||
for {
|
||||
raw, rev, err := k.st.get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var v PinsValue
|
||||
if raw != nil {
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return fmt.Errorf("the pins of %s do not read: %w", key, err)
|
||||
}
|
||||
}
|
||||
next, write := change(v, raw != nil)
|
||||
if !write {
|
||||
return nil
|
||||
}
|
||||
value, _ := json.Marshal(next)
|
||||
if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// held says the revision's object is kept.
|
||||
func (k *pinKeeper) held(revision int64) (bool, error) {
|
||||
_, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision))
|
||||
if errors.Is(err, assigngraph.ErrNoObject) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding
|
||||
// pin-missing naming the record.
|
||||
func (k *pinKeeper) pin(p mdseat.Pins) error {
|
||||
for _, r := range p.Assignment {
|
||||
held, err := k.held(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) {
|
||||
if slices.Contains(v.Records, p.Record) && (held || v.Missing) {
|
||||
return v, false
|
||||
}
|
||||
if !slices.Contains(v.Records, p.Record) {
|
||||
v.Records = append(v.Records, p.Record)
|
||||
slices.Sort(v.Records)
|
||||
}
|
||||
v.Missing = v.Missing || !held
|
||||
return v, true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
k.missing(r, p.Record)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// free removes record from each assignment graph revision of p; a freed applied twice changes nothing.
|
||||
func (k *pinKeeper) free(p mdseat.Pins) error {
|
||||
for _, r := range p.Assignment {
|
||||
err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) {
|
||||
i := slices.Index(v.Records, p.Record)
|
||||
if !found || i < 0 {
|
||||
return v, false
|
||||
}
|
||||
v.Records = slices.Delete(v.Records, i, i+1)
|
||||
return v, true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missing writes the finding pin-missing, scope the assigner, id the revision.
|
||||
func (k *pinKeeper) missing(revision int64, record string) {
|
||||
if k.findings.Put == nil {
|
||||
k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record)
|
||||
return
|
||||
}
|
||||
o := assign.Observed{Scope: assignment.Module(), ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind,
|
||||
Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision),
|
||||
Headline: "A delivery record pins a revision the assigner no longer holds",
|
||||
Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool),
|
||||
Source: assignment.Module()}
|
||||
if err := k.findings.Put(o.Finding(k.now())); err != nil {
|
||||
k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err)
|
||||
}
|
||||
}
|
||||
|
||||
// made writes the key made once, create-only; one there already stays.
|
||||
func (k *pinKeeper) made() error {
|
||||
raw, _, err := k.st.get(pinsMadeKey)
|
||||
if err != nil || raw != nil {
|
||||
return err
|
||||
}
|
||||
value, _ := json.Marshal(k.now().UTC())
|
||||
if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// timeAt is the time a key holds; false when it holds none.
|
||||
func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) {
|
||||
raw, _, err := k.st.get(key)
|
||||
if err != nil || raw == nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
var t time.Time
|
||||
if err := json.Unmarshal(raw, &t); err != nil {
|
||||
return t, false, fmt.Errorf("the key %s does not read: %w", key, err)
|
||||
}
|
||||
return t, true, nil
|
||||
}
|
||||
|
||||
// stated keeps the at of a pins-stated applied.
|
||||
func (k *pinKeeper) stated(at time.Time) error {
|
||||
for {
|
||||
raw, rev, err := k.st.get(pinsStatedKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var was time.Time
|
||||
if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) {
|
||||
return nil
|
||||
}
|
||||
value, _ := json.Marshal(at.UTC())
|
||||
if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// statedSinceMade says a pins-stated said after the state was made was applied.
|
||||
func (k *pinKeeper) statedSinceMade() (bool, error) {
|
||||
made, ok, err := k.timeAt(pinsMadeKey)
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
at, ok, err := k.timeAt(pinsStatedKey)
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
return at.After(made), nil
|
||||
}
|
||||
|
||||
// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied.
|
||||
func (k *pinKeeper) hear(m jetstream.Msg) {
|
||||
err := k.apply(m.Subject(), m.Data())
|
||||
if err != nil {
|
||||
k.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
caught := false
|
||||
if meta, merr := m.Metadata(); merr == nil {
|
||||
caught = meta.NumPending == 0
|
||||
}
|
||||
k.mu.Lock()
|
||||
was := k.caught
|
||||
k.caught = caught
|
||||
k.mu.Unlock()
|
||||
if caught && (!was || m.Subject() != mdseat.SubjectPinned) {
|
||||
k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up
|
||||
}
|
||||
}
|
||||
|
||||
// apply takes one event's body.
|
||||
func (k *pinKeeper) apply(subject string, data []byte) error {
|
||||
switch subject {
|
||||
case mdseat.SubjectPinned, mdseat.SubjectFreed:
|
||||
var p mdseat.Pins
|
||||
if err := json.Unmarshal(data, &p); err != nil || p.Record == "" {
|
||||
k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err)
|
||||
return nil
|
||||
}
|
||||
if subject == mdseat.SubjectPinned {
|
||||
return k.pin(p)
|
||||
}
|
||||
return k.free(p)
|
||||
case mdseat.SubjectPinsStated:
|
||||
var s mdseat.PinsStatedEvent
|
||||
if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() {
|
||||
k.logf("%s holds no at, and is not taken: %v", subject, err)
|
||||
return nil
|
||||
}
|
||||
return k.stated(s.At)
|
||||
}
|
||||
return nil // heads-replaced: a candidate's revision stops being current, judged after
|
||||
}
|
||||
|
||||
// judge removes every revision neither pinned nor current, when the rule allows a removal at all.
|
||||
func (k *pinKeeper) judge() {
|
||||
k.mu.Lock()
|
||||
caught := k.caught
|
||||
k.mu.Unlock()
|
||||
if !caught {
|
||||
return
|
||||
}
|
||||
ok, err := k.statedSinceMade()
|
||||
if err != nil {
|
||||
k.logf("the pins are not judged: %v", err)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
return // no pins-stated applied since the state was made: a pin said before may not be heard yet
|
||||
}
|
||||
if err := k.remove(); err != nil {
|
||||
k.logf("the revisions nothing pins are not removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted.
|
||||
func (k *pinKeeper) remove() error {
|
||||
names, err := k.objects.Names(k.ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var kept []int64
|
||||
for _, n := range names {
|
||||
if r, ok := pinRevision(n); ok {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil
|
||||
}
|
||||
current := slices.Max(kept)
|
||||
listed := map[string]bool{}
|
||||
removed := 0
|
||||
for _, r := range kept {
|
||||
raw, _, err := k.st.get(pinKey(r))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var v PinsValue
|
||||
if raw != nil {
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return fmt.Errorf("the pins of %d do not read: %w", r, err)
|
||||
}
|
||||
}
|
||||
if r == current || len(v.Records) > 0 {
|
||||
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
|
||||
return fmt.Errorf("revision %d: %w", r, err)
|
||||
}
|
||||
if raw != nil {
|
||||
if err := k.st.drop(pinKey(r)); err != nil {
|
||||
return fmt.Errorf("the pins of %d: %w", r, err)
|
||||
}
|
||||
}
|
||||
removed++
|
||||
}
|
||||
if removed == 0 {
|
||||
return nil
|
||||
}
|
||||
// A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235).
|
||||
for _, n := range names {
|
||||
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
}
|
||||
for _, n := range names {
|
||||
digest, ok := strings.CutPrefix(n, "entry.")
|
||||
if !ok || listed[digest] {
|
||||
continue
|
||||
}
|
||||
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, n); err != nil {
|
||||
return fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
}
|
||||
k.logf("%d assignment graph revisions no record pins and not current are removed", removed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hearPins registers the seat mesh-delivery's four events on the assigner's hub.
|
||||
func hearPins(hub *process.EventHub, k *pinKeeper) error {
|
||||
for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} {
|
||||
if err := hub.Handle(s, k.hear); err != nil {
|
||||
return fmt.Errorf("%s: %w", s, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinsTools are assigner_pins and assigner_unpin over the runtime's states.
|
||||
func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " +
|
||||
"the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " +
|
||||
"was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.",
|
||||
Run: func(map[string]any) (any, error) { return readPins(st) }},
|
||||
{Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " +
|
||||
"hand-act log first. The removal rule then applies.",
|
||||
Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id",
|
||||
"why": "why it is removed by hand"}, []string{"revision", "record", "why"}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
record, _ := a["record"].(string)
|
||||
why, _ := a["why"].(string)
|
||||
return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a))
|
||||
}}}
|
||||
}
|
||||
|
||||
// readPins is what assigner_pins answers.
|
||||
func readPins(st store) (any, error) {
|
||||
ks, err := st.keys(pinsState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(ks)
|
||||
revisions := map[string]PinsValue{}
|
||||
out := map[string]any{"revisions": revisions}
|
||||
var newest int64
|
||||
bytes := 0
|
||||
for _, key := range ks {
|
||||
raw, _, found, err := st.get(pinsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
bytes += len(key) + len(raw)
|
||||
if r, ok := pinRevision(key); ok {
|
||||
var v PinsValue
|
||||
if json.Unmarshal(raw, &v) == nil {
|
||||
revisions[key] = v
|
||||
}
|
||||
newest = max(newest, r)
|
||||
continue
|
||||
}
|
||||
out[key] = json.RawMessage(raw)
|
||||
}
|
||||
out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes}
|
||||
if newest > 0 {
|
||||
out["newest-pinned"] = newest
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unpin removes one record from one revision's list, the hand act recorded first.
|
||||
func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) {
|
||||
r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64)
|
||||
switch {
|
||||
case err != nil || r <= 0:
|
||||
return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision)
|
||||
case record == "" || why == "":
|
||||
return nil, errors.New("name the record and say why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("an unpin names who asks it, and the runtime proved no caller")
|
||||
}
|
||||
key := pinKey(r)
|
||||
raw, _, found, err := st.get(pinsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var v PinsValue
|
||||
if found {
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
i := slices.Index(v.Records, record)
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool)
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record,
|
||||
Why: why + " (" + by + ")", Cause: "unpin"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err)
|
||||
}
|
||||
v.Records = slices.Delete(v.Records, i, i+1)
|
||||
value, _ := json.Marshal(v)
|
||||
if err := st.put(pinsState, key, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool
|
||||
// process: the assigner's own state, heard as it changes, never on a timer.
|
||||
func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) {
|
||||
w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut {
|
||||
k.judge()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
|
||||
func inUseOf(w *world) func(module, fp string) bool {
|
||||
return func(module, fp string) bool {
|
||||
for _, want := range w.wantsOf()[module] {
|
||||
if want.Fp == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, mods := range w.runningNow() {
|
||||
if r, ok := mods[module]; ok && r.Fingerprint == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func pinsBucket() string { return assignment.Module() + "_" + pinsState }
|
||||
@@ -1,119 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
|
||||
|
||||
// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until
|
||||
// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a
|
||||
// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing,
|
||||
// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays.
|
||||
func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) {
|
||||
objects := memObjects{}
|
||||
entry := func(module, fp string) string {
|
||||
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
|
||||
d := assigngraph.EntryDigest(e)
|
||||
raw, _ := json.Marshal(e)
|
||||
objects[assigngraph.EntryObject(d)] = raw
|
||||
return d
|
||||
}
|
||||
shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1")
|
||||
revision := func(n int64, digests ...string) {
|
||||
rev := meshwants.AssignmentRevision{Revision: n}
|
||||
for _, d := range digests {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
|
||||
}
|
||||
raw, _ := json.Marshal(rev)
|
||||
objects[assigngraph.TrunkObject(n)] = raw
|
||||
}
|
||||
// A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept.
|
||||
cand := entry("d", "fp-d2")
|
||||
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
|
||||
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
|
||||
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
|
||||
revision(1, shared, old, running)
|
||||
revision(2, shared, old)
|
||||
revision(3, shared)
|
||||
var found []mcseat.Finding
|
||||
clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock },
|
||||
logf: t.Logf,
|
||||
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" },
|
||||
findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}}
|
||||
if err := k.made(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
say := func(subject string, body any) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(body)
|
||||
if err := k.apply(subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.judge()
|
||||
}
|
||||
pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} }
|
||||
has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok }
|
||||
|
||||
k.caught = true
|
||||
say(mdseat.SubjectPinned, pins("r1", 1))
|
||||
say(mdseat.SubjectPinned, pins("r1", 1))
|
||||
say(mdseat.SubjectPinned, pins("r2", 1))
|
||||
say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed
|
||||
if !has(2) {
|
||||
t.Fatal("revision 2 was removed before a pins-stated said after the state was made")
|
||||
}
|
||||
k.caught = false
|
||||
say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)})
|
||||
if !has(2) {
|
||||
t.Fatal("revision 2 was removed while the consumer was behind")
|
||||
}
|
||||
k.caught = true
|
||||
say(mdseat.SubjectFreed, pins("r1", 1))
|
||||
if has(2) {
|
||||
t.Error("revision 2, neither pinned nor current, is kept")
|
||||
}
|
||||
if !has(1) || !has(3) {
|
||||
t.Error("revision 1 (pinned by r2) or 3 (current) was removed")
|
||||
}
|
||||
raw, _, _ := k.st.get(pinKey(1))
|
||||
if string(raw) != `{"records":["r2"]}` {
|
||||
t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw)
|
||||
}
|
||||
say(mdseat.SubjectFreed, pins("r2", 1))
|
||||
if has(1) {
|
||||
t.Error("revision 1 is kept once both records freed it")
|
||||
}
|
||||
if _, ok := objects[assigngraph.EntryObject(old)]; ok {
|
||||
t.Error("the entry no kept revision lists and nothing names is kept")
|
||||
}
|
||||
if _, ok := objects[assigngraph.EntryObject(running)]; !ok {
|
||||
t.Error("the entry of a running build is removed")
|
||||
}
|
||||
if _, ok := objects[assigngraph.EntryObject(shared)]; !ok {
|
||||
t.Error("the entry the current revision lists is removed")
|
||||
}
|
||||
if _, ok := objects[assigngraph.EntryObject(cand)]; !ok {
|
||||
t.Error("the entry a run's kept candidate lists is removed")
|
||||
}
|
||||
say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{})
|
||||
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok {
|
||||
t.Error("the gate's heads-replaced dropped a run's candidate")
|
||||
}
|
||||
say(mdseat.SubjectPinned, pins("r3", 2))
|
||||
raw, _, _ = k.st.get(pinKey(2))
|
||||
if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" {
|
||||
t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found)
|
||||
}
|
||||
}
|
||||
@@ -14,20 +14,22 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as
|
||||
// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and
|
||||
// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written
|
||||
// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing.
|
||||
// The wants follow a passed pipeline (design 47 H §750): the runner says each pipeline's end once, per source revision, as
|
||||
// pipeline-ended on `pipeline-ended.<branch>.<source>`, naming per mover its fingerprint, its released version and its tier
|
||||
// (§746). The assigner keeps, in its state `written` (G §352), one key per module and watched branch: the newest released
|
||||
// version of the module on that branch whose pipeline passed, with the pipeline and the mover's rank in it. A newer version
|
||||
// replaces it by compare-and-set; an older one, the same one again, a failed pipeline and a run on request write nothing.
|
||||
|
||||
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
|
||||
const (
|
||||
@@ -35,23 +37,17 @@ const (
|
||||
writtenTool = "assigner_written"
|
||||
)
|
||||
|
||||
// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes.
|
||||
const (
|
||||
pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
|
||||
groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>"
|
||||
)
|
||||
// pipelineEndedSubject is the runner's event the assigner consumes.
|
||||
const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
|
||||
|
||||
// End is one value of the state written: the newest end of a module's pipeline on a watched branch.
|
||||
// End is one value of the state written: the newest version of a module released on a watched branch whose pipeline passed.
|
||||
type End struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Source int64 `json:"source"`
|
||||
EndCount int64 `json:"end-count"`
|
||||
Fp string `json:"fp"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
// Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed).
|
||||
Passed bool `json:"passed"`
|
||||
// Written: the wants were written from this end.
|
||||
Written bool `json:"written,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Version string `json:"version"`
|
||||
// Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750).
|
||||
Rank int64 `json:"rank"`
|
||||
}
|
||||
|
||||
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
|
||||
@@ -59,15 +55,10 @@ func endKey(module, branch string) string {
|
||||
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
|
||||
}
|
||||
|
||||
// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again.
|
||||
func (e End) newer(source, count int64) bool {
|
||||
return source > e.Source || (source == e.Source && count > e.EndCount)
|
||||
}
|
||||
|
||||
// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed
|
||||
// optional step holds nothing).
|
||||
func requiredPassed(p mpr.PipelineEnded) bool {
|
||||
for _, s := range p.Steps {
|
||||
// moverPassed says whether no required step of a mover stands failed, unasked or unended: a required step ending "nothing to
|
||||
// run" counts as passed, as a bypassed one does (design 47 H §746, H-A-3); a failed optional step holds nothing (G §30).
|
||||
func moverPassed(m mpr.MoverEnded) bool {
|
||||
for _, s := range m.Steps {
|
||||
if s.Status != mpr.StepResultStatusRequired {
|
||||
continue
|
||||
}
|
||||
@@ -78,7 +69,7 @@ func requiredPassed(p mpr.PipelineEnded) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return p.Outcome == mpr.PipelineEndedOutcomePassed
|
||||
return true
|
||||
}
|
||||
|
||||
// endStore is the kept state written: the bus bucket, or a test's.
|
||||
@@ -139,128 +130,182 @@ func isWrongSequence(err error) bool {
|
||||
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
|
||||
}
|
||||
|
||||
// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations
|
||||
// (design 47 G §95, §353).
|
||||
// pipelineEnds follows the runner's ends; a nil *pipelineEnds keeps none, and no want is written from it.
|
||||
type pipelineEnds struct {
|
||||
st endStore
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353).
|
||||
// on says the first registration was heard (design 47 G §353), said once in the journal.
|
||||
on atomic.Bool
|
||||
mu sync.Mutex
|
||||
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
|
||||
// 47 G §14).
|
||||
firsts map[string]string
|
||||
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
|
||||
// runEnded is told a run on request's pipeline-ended, its group `run-<id>`, to drop the run's candidate (A-235, H-A-58); nil
|
||||
// drops nothing.
|
||||
runEnded func(group string) error
|
||||
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
|
||||
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
|
||||
left atomic.Pointer[func(pipeline string) error]
|
||||
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
|
||||
onBranches func([]string)
|
||||
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
|
||||
trunk func() string
|
||||
// runner is the module holding the seat mesh-pipeline-runner, as the last pass read it from the acting edges (A-334); nil
|
||||
// until a pass read the graph.
|
||||
runner atomic.Pointer[string]
|
||||
// held are the group members' pipeline-ended heard before a pass read the graph, by their body: unacknowledged, so the
|
||||
// bus keeps them, and taken once the runner is read (A-334), never dropped for the runner not being known yet.
|
||||
held map[string]jetstream.Msg
|
||||
}
|
||||
|
||||
// setRunner keeps the module holding the seat mesh-pipeline-runner, read by the pass from the acting edges (A-334); the
|
||||
// first one takes the group members' ends held until the graph was read.
|
||||
func (e *pipelineEnds) setRunner(module string) {
|
||||
e.mu.Lock()
|
||||
first := e.runner.Swap(&module) == nil
|
||||
held := e.held
|
||||
e.held = nil
|
||||
e.mu.Unlock()
|
||||
if !first {
|
||||
return
|
||||
}
|
||||
for _, m := range held {
|
||||
e.hearPipeline(m)
|
||||
}
|
||||
}
|
||||
|
||||
// holdUntilRead keeps m, a group member's pipeline-ended, until a pass reads the runner from the graph; false when the
|
||||
// runner is read already, so m is taken now.
|
||||
func (e *pipelineEnds) holdUntilRead(m jetstream.Msg) bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.runner.Load() != nil {
|
||||
return false
|
||||
}
|
||||
if e.held == nil {
|
||||
e.held = map[string]jetstream.Msg{}
|
||||
}
|
||||
e.held[string(m.Data())] = m
|
||||
e.logf("%s is a group member's end heard before the assignment graph is read: held until a pass reads which module is the runner (A-334)", m.Subject())
|
||||
return true
|
||||
}
|
||||
|
||||
// isRunner says whether module holds the seat mesh-pipeline-runner, as the last pass read it.
|
||||
func (e *pipelineEnds) isRunner(module string) bool {
|
||||
r := e.runner.Load()
|
||||
return r != nil && *r != "" && *r == module
|
||||
// now is the clock of the tier progress (tiers.go); nil is time.Now.
|
||||
now func() time.Time
|
||||
// releasedOf is the version released of module's build at fp, matched by the parts its build.ended made (world.releasedOf):
|
||||
// a mover whose version was released before its pipeline passed gets its want at the pass (design 47 H2 H-A-490). nil
|
||||
// matches none.
|
||||
releasedOf func(module, fp string) (string, bool)
|
||||
}
|
||||
|
||||
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
|
||||
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
|
||||
}
|
||||
|
||||
// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error {
|
||||
key := endKey(p.Module, p.Branch)
|
||||
for {
|
||||
raw, rev, err := e.st.get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
|
||||
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
|
||||
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). A mover whose build's version was
|
||||
// released before the pass is taken at the pass with that version (H-A-490). It says whether any key changed.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
}
|
||||
// The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not
|
||||
// know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress.
|
||||
if _, err := pipelinekey.Parse(p.Pipeline); err != nil {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
|
||||
return false, nil
|
||||
}
|
||||
changed := false
|
||||
var kept []tierMover
|
||||
versioned := map[string]bool{}
|
||||
defer func() {
|
||||
if err := e.startTiers(p.Pipeline, p.Source, kept, versioned); err != nil {
|
||||
e.logf("the tier progress of %s is not written: %v", p.Pipeline, err)
|
||||
}
|
||||
if raw != nil {
|
||||
var kept End
|
||||
if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) {
|
||||
return nil // an older end, or this one again
|
||||
}()
|
||||
for _, m := range p.Movers {
|
||||
if m.Fingerprint == "" || !moverPassed(m) {
|
||||
continue
|
||||
}
|
||||
if m.Version == "" && e.releasedOf != nil {
|
||||
// A version released before its pipeline passed (an again pipeline on a built, published key) is never written
|
||||
// on the mover, nor its end said again: it is judged at the pass, matched by its build's parts (H-A-490).
|
||||
if v, ok := e.releasedOf(m.Module, m.Fingerprint); ok {
|
||||
e.logf("%s: %s passed; its version %s was released before the pass, taken now", p.Pipeline, m.Module, v)
|
||||
m.Version = v
|
||||
}
|
||||
}
|
||||
end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fingerprint, Commit: p.Commit, Passed: passed, Why: p.Why}
|
||||
value, err := json.Marshal(end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
|
||||
if m.Version == "" {
|
||||
// The runner writes a mover's version at its release's version-released, and says this end again once it does
|
||||
// (H-A-270 (1), (2)): the skip is said, never silent.
|
||||
e.logf("%s: %s passed with no version yet: taken again when its version is written", p.Pipeline, m.Module)
|
||||
// It holds its tier in the progress until its version is written (H-A-277 (1)).
|
||||
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1), NoVersion: true})
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
e.wake()
|
||||
return nil
|
||||
versioned[m.Module] = true
|
||||
key := endKey(m.Module, p.Branch)
|
||||
for {
|
||||
raw, rev, err := e.st.get(key)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if raw != nil {
|
||||
var kept End
|
||||
if err := json.Unmarshal(raw, &kept); err == nil && compareVersion(m.Version, kept.Version) <= 0 {
|
||||
break // an older version, or this one again
|
||||
}
|
||||
}
|
||||
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
changed = true
|
||||
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1)})
|
||||
break
|
||||
}
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp.
|
||||
func (e *pipelineEnds) markWritten(module, branch, fp string) {
|
||||
key := endKey(module, branch)
|
||||
for {
|
||||
raw, rev, err := e.st.get(key)
|
||||
var kept End
|
||||
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written {
|
||||
return
|
||||
}
|
||||
kept.Written = true
|
||||
value, _ := json.Marshal(kept)
|
||||
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
e.logf("the end of %s on %s is not marked written: %v", module, branch, err)
|
||||
}
|
||||
return
|
||||
// endOf is the end kept for module on branch: the newest version released there whose pipeline passed.
|
||||
func (e *pipelineEnds) endOf(module, branch string) (End, bool) {
|
||||
if e == nil {
|
||||
return End{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// passedAt says whether the newest end kept for module on branch passed at fp.
|
||||
func (e *pipelineEnds) passedAt(module, branch, fp string) bool {
|
||||
raw, _, err := e.st.get(endKey(module, branch))
|
||||
var kept End
|
||||
return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp
|
||||
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp == "" {
|
||||
return End{}, false
|
||||
}
|
||||
return kept, true
|
||||
}
|
||||
|
||||
// target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline
|
||||
// passed, with the pipeline and the mover's rank; false when none is kept, or its pipeline's tier progress has not reached its
|
||||
// rank (§751).
|
||||
func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) {
|
||||
end, ok := e.endOf(module, e.branchOf(module, branch))
|
||||
if !ok || !e.tierWritten(end) {
|
||||
return assign.Target{}, false
|
||||
}
|
||||
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true
|
||||
}
|
||||
|
||||
// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the
|
||||
// run's candidate assignment graph (A-235, H-A-58); a run delivers nothing (design 47 G §29, §64). A pipeline ended superseded or
|
||||
// cancelled has its composition steps leave the runs they read (G §723).
|
||||
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
|
||||
p, err := mprc.PipelineEnded(m.Data())
|
||||
if err != nil {
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if left := e.left.Load(); left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
|
||||
if err := (*left)(p.Pipeline); err != nil {
|
||||
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.Run != "" {
|
||||
if e.runEnded != nil {
|
||||
if err := e.runEnded(runGroupPrefix + p.Run); err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
changed, err := e.take(p)
|
||||
if err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if changed {
|
||||
e.wake()
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// hearEnds registers pipeline-ended on the assigner's hub.
|
||||
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
|
||||
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
|
||||
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
|
||||
@@ -279,84 +324,35 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, but the runner's: in a group
|
||||
// take-in that changes the runner it is the acting part the other members wait on, so it goes first, wanted on its own end
|
||||
// once that passed, ahead of the group (A-334, as the node-engine does, §268). A run on request delivers nothing (design 47 G
|
||||
// §29, §64).
|
||||
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
|
||||
p, err := mprc.PipelineEnded(m.Data())
|
||||
switch {
|
||||
case err != nil:
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
case p.Run == "" && p.Group != "" && e.holdUntilRead(m):
|
||||
return
|
||||
case p.Run != "" || (p.Group != "" && !e.isRunner(p.Module)):
|
||||
default:
|
||||
if err := e.take(p, requiredPassed(p)); err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-<id>`) delivers
|
||||
// nothing: it drops the run's candidate assignment graph (A-235).
|
||||
func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
|
||||
g, err := mprc.GroupEnded(m.Data())
|
||||
if err != nil {
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(g.Group, runGroupPrefix) {
|
||||
if e.runEnded != nil {
|
||||
if err := e.runEnded(g.Group); err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
for _, p := range g.Pipelines {
|
||||
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// hearEnds registers the two events on the assigner's hub.
|
||||
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
|
||||
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
|
||||
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
|
||||
}
|
||||
if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil {
|
||||
return fmt.Errorf("%s: %w", groupEndedSubject, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writtenTools is the tool assigner_written: every key of the state written, with its end.
|
||||
func writtenTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the newest version released whose pipeline passed, with its pipeline and rank; per pipeline whose wants are written tier by tier, its tier progress (tier, since, movers); with why, why each want is what it is (as assigner_why).",
|
||||
Input: schema(map[string]string{"why": "true: why each want is what it is", "module": "with why: a module's name", "node": "with why: a node's name"}, nil, "why"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if args["why"] == true || args["why"] == "true" {
|
||||
return whyTools(st)[0].Run(args) // a want's why (why.go), until the manifest naming assigner_why is taken in
|
||||
}
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(ks)
|
||||
out := map[string]End{}
|
||||
out := map[string]any{}
|
||||
for _, k := range ks {
|
||||
if strings.HasPrefix(k, whyPrefix) {
|
||||
continue // a want's why (why.go), read by assigner_why
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
if isTierKey(k) { // a pipeline's tier progress (tiers.go)
|
||||
var t tierProgress
|
||||
if json.Unmarshal(raw, &t) == nil {
|
||||
out[k] = t
|
||||
}
|
||||
continue
|
||||
}
|
||||
var e End
|
||||
if json.Unmarshal(raw, &e) == nil {
|
||||
out[k] = e
|
||||
@@ -366,56 +362,6 @@ func writtenTools(st store) []stdio.Tool {
|
||||
}}}
|
||||
}
|
||||
|
||||
// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one.
|
||||
func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string {
|
||||
if !e.following() {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, b := range placed[module] {
|
||||
b = e.branchOf(module, b)
|
||||
if !seen[b] {
|
||||
seen[b] = true
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = append(out, e.branchOf(module, ""))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already
|
||||
// (a node placed later, a module taken away).
|
||||
func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool {
|
||||
if !e.following() {
|
||||
return true
|
||||
}
|
||||
for _, w := range wanted {
|
||||
if w.Fp == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, b := range branches {
|
||||
if !e.passedAt(module, b, fp) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// wrote marks the end the wants of module were written from.
|
||||
func (e *pipelineEnds) wrote(module, fp string, branches []string) {
|
||||
if !e.following() {
|
||||
return
|
||||
}
|
||||
for _, b := range branches {
|
||||
e.markWritten(module, b, fp)
|
||||
}
|
||||
}
|
||||
|
||||
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
|
||||
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
|
||||
|
||||
@@ -437,7 +383,7 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
|
||||
prefix := keys.EscapeName(reg.Module) + "."
|
||||
for _, k := range kept {
|
||||
rest, ok := strings.CutPrefix(k, prefix)
|
||||
if !ok {
|
||||
if !ok || isTierKey(k) || strings.HasPrefix(k, whyPrefix) {
|
||||
continue
|
||||
}
|
||||
branch, err := keys.UnescapeName(rest)
|
||||
@@ -456,32 +402,3 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
|
||||
}
|
||||
e.wake()
|
||||
}
|
||||
|
||||
// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb
|
||||
// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own
|
||||
// goroutine, the wants keep the trunk-only form until a registration is heard.
|
||||
func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error {
|
||||
regs := mcc.NewRegistrations(call)
|
||||
regs.Follower.Logf = e.logf
|
||||
regs.Follower.Changed = func(module string, value json.RawMessage, gone bool, _ time.Time) {
|
||||
reg := mpcat.Registration{Module: module}
|
||||
if !gone {
|
||||
_ = json.Unmarshal(value, ®)
|
||||
}
|
||||
e.registered(reg, gone)
|
||||
}
|
||||
if err := hub.Handle(mpcat.SubjectRegistrationChanged, func(m jetstream.Msg) {
|
||||
if err := regs.Hear(ctx, m.Data()); err != nil {
|
||||
e.logf("%s: %v", m.Subject(), err)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
if err := regs.Start(ctx); err != nil && ctx.Err() == nil {
|
||||
e.logf("the first view of the registrations is not answered: %v", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,34 +4,25 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
|
||||
mws "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// memEnds is endStore over a map, with revisions.
|
||||
@@ -71,239 +62,191 @@ func (m *heardMsg) Subject() string { return m.subject }
|
||||
func (m *heardMsg) Ack() error { m.acked = true; return nil }
|
||||
func (m *heardMsg) Nak() error { m.naked = true; return nil }
|
||||
|
||||
func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded {
|
||||
return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps,
|
||||
Pipeline: branch + "." + module, Commit: "c"}
|
||||
// mover is one mover at its pipeline's end, its required unit step ending outcome.
|
||||
func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnded {
|
||||
return mpr.MoverEnded{Module: module, Fingerprint: fp, Version: version, Tier: tier,
|
||||
Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}}
|
||||
}
|
||||
|
||||
func step(status, outcome string) mpr.StepResult {
|
||||
return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome}
|
||||
// ended is the pipeline-ended of the take-in of source on branch, its movers given.
|
||||
func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded {
|
||||
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers,
|
||||
Pipeline: fmt.Sprintf("%s.%d", branch, source)}
|
||||
}
|
||||
|
||||
// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch.
|
||||
func TestAWantFollowsAPassedPipeline(t *testing.T) {
|
||||
const fp1, fp2 = "fp1", "fp2"
|
||||
// A6/AC1 to AC3 (design 47 H §750, H-A-3): a passed pipeline keeps each mover's released version as the newest of its module on
|
||||
// the branch, with the pipeline and the mover's tier; a failed pipeline, a mover whose required step failed, one that released
|
||||
// no version, an older version, the same one again and a run on request keep nothing.
|
||||
func TestThePassedVersionIsKept(t *testing.T) {
|
||||
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
|
||||
req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional
|
||||
ok, failed, nothing := mpr.StepResultOutcomePassed, mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeNothingToRun
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ends []mpr.PipelineEnded
|
||||
group *mpr.GroupEnded
|
||||
module string
|
||||
branch string
|
||||
fp string
|
||||
may bool
|
||||
name string
|
||||
ends []mpr.PipelineEnded
|
||||
want *End // nil: none kept for web on main
|
||||
}{
|
||||
{"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false},
|
||||
{"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
|
||||
{"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true},
|
||||
{"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false},
|
||||
{"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false},
|
||||
{"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true},
|
||||
{"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
|
||||
{"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true},
|
||||
{"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false},
|
||||
{"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true},
|
||||
{"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false},
|
||||
{"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}},
|
||||
{"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok),
|
||||
mover("db", "fp9", "0.3.0", 1, failed))}, nil},
|
||||
{"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil},
|
||||
{"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil},
|
||||
{"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)),
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
|
||||
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.again.2"
|
||||
return p
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.later.2"
|
||||
return p
|
||||
}()}, nil},
|
||||
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Run = "r1"
|
||||
return p
|
||||
}()}, nil},
|
||||
} {
|
||||
woken := 0
|
||||
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
|
||||
e.on.Store(true)
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
for _, p := range c.ends {
|
||||
if p.Group != "" {
|
||||
continue // heard and ignored, as hearPipeline does
|
||||
}
|
||||
if err := e.take(p, requiredPassed(p)); err != nil {
|
||||
t.Fatal(err)
|
||||
if _, err := e.take(p); err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
if g := c.group; g != nil {
|
||||
for _, p := range g.Pipelines {
|
||||
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, kept := e.endOf("web", "main")
|
||||
if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 {
|
||||
t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks)
|
||||
}
|
||||
if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may {
|
||||
t.Errorf("%s: may want = %v, want %v", c.name, got, c.may)
|
||||
switch {
|
||||
case c.want == nil && kept:
|
||||
t.Errorf("%s: kept %+v", c.name, got)
|
||||
case c.want != nil && (!kept || got != *c.want):
|
||||
t.Errorf("%s: kept %+v (%v), want %+v", c.name, got, kept, *c.want)
|
||||
}
|
||||
}
|
||||
// A run on request's group-ended (group run-<id>, A-235) writes no want and drops the run's candidate; a drop that fails is
|
||||
// heard again.
|
||||
{
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
e.on.Store(true)
|
||||
var dropped []string
|
||||
fail := false
|
||||
e.runEnded = func(group string) error {
|
||||
dropped = append(dropped, group)
|
||||
if fail {
|
||||
return fmt.Errorf("the bucket is not open")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed,
|
||||
Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}})
|
||||
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"}
|
||||
e.hearGroup(m)
|
||||
if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked {
|
||||
t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked",
|
||||
e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked)
|
||||
}
|
||||
fail = true
|
||||
m = &heardMsg{data: raw, subject: m.subject}
|
||||
e.hearGroup(m)
|
||||
if !m.naked {
|
||||
t.Error("a run's candidate not dropped is not heard again")
|
||||
// The same end heard again changes nothing and wakes nothing; a run on request's end drops its candidate.
|
||||
woken := 0
|
||||
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
|
||||
var dropped []string
|
||||
e.runEnded = func(group string) error { dropped = append(dropped, group); return nil }
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
for range 2 {
|
||||
raw, _ := json.Marshal(p)
|
||||
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"}
|
||||
e.hearPipeline(m)
|
||||
if !m.acked {
|
||||
t.Error("the end was not acknowledged")
|
||||
}
|
||||
}
|
||||
// A-334 (§29, §65, §268): in a group take-in that changes the runner, the runner's member (the holder of the seat
|
||||
// mesh-pipeline-runner, read from the acting edges) is wanted on its own pipeline-ended once it passed, ahead of the group;
|
||||
// the other members wait for group-ended.
|
||||
{
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
e.on.Store(true)
|
||||
heard := func(p mpr.PipelineEnded) *heardMsg {
|
||||
p.Group = "g"
|
||||
raw, _ := json.Marshal(p)
|
||||
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.x"}
|
||||
e.hearPipeline(m)
|
||||
return m
|
||||
}
|
||||
hear := func(p mpr.PipelineEnded) {
|
||||
if m := heard(p); !m.acked {
|
||||
t.Errorf("the pipeline-ended of %s is not acknowledged", p.Module)
|
||||
}
|
||||
}
|
||||
// The runner's end heard before a pass read the graph is held, unacknowledged, and taken once the runner is read.
|
||||
early := heard(pipe("runner", "main", 5, 1, "fp-runner", pass, step(req, "passed")))
|
||||
if early.acked || early.naked || e.mayWant("runner", "fp-runner", []string{"main"}, nil) {
|
||||
t.Fatalf("the runner's end before the graph is read: acked %v, naked %v, wanted %v; want it held", early.acked, early.naked,
|
||||
e.mayWant("runner", "fp-runner", []string{"main"}, nil))
|
||||
}
|
||||
e.setRunner(assigngraph.Runner([]mws.AssignmentEntry{{Node: "ace", Module: "web", Edges: []mws.Edge{
|
||||
{Kind: assigngraph.EdgeActing, Node: "ace", Module: "runner", Seat: mpr.Seat, Step: "pipeline"}}}}))
|
||||
if !early.acked {
|
||||
t.Error("the runner's end held until the graph was read is not taken once it is")
|
||||
}
|
||||
hear(pipe("web", "main", 5, 1, fp1, pass, step(req, "passed")))
|
||||
if !e.mayWant("runner", "fp-runner", []string{"main"}, nil) || e.mayWant("web", fp1, []string{"main"}, nil) {
|
||||
t.Errorf("a group changing the runner: may want the runner %v, web %v; want the runner ahead of the group and web not yet",
|
||||
e.mayWant("runner", "fp-runner", []string{"main"}, nil), e.mayWant("web", fp1, []string{"main"}, nil))
|
||||
}
|
||||
hear(pipe("runner", "main", 6, 1, "fp-runner-2", fail, step(req, "failed")))
|
||||
if e.mayWant("runner", "fp-runner-2", []string{"main"}, nil) {
|
||||
t.Error("the runner's member that failed is wanted")
|
||||
}
|
||||
}
|
||||
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
e.on.Store(true)
|
||||
if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) {
|
||||
t.Error("a want naming the fingerprint was held")
|
||||
}
|
||||
// markWritten keeps the end and says it was written from; a newer end replaces it unwritten.
|
||||
st := newMemEnds()
|
||||
e = newPipelineEnds(st, func() {}, t.Logf)
|
||||
e.on.Store(true)
|
||||
_ = e.take(pipe("web", "main", 5, 1, fp1, pass), true)
|
||||
e.wrote("web", fp1, []string{"main"})
|
||||
var end End
|
||||
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
|
||||
if !end.Written || end.Source != 5 {
|
||||
t.Errorf("end = %+v, want written at source 5", end)
|
||||
}
|
||||
_ = e.take(pipe("web", "main", 6, 1, fp2, pass), true)
|
||||
end = End{}
|
||||
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
|
||||
if end.Written || end.Source != 6 {
|
||||
t.Errorf("a newer end = %+v, want unwritten at source 6", end)
|
||||
run := p
|
||||
run.Run = "r1"
|
||||
raw, _ := json.Marshal(run)
|
||||
e.hearPipeline(&heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"})
|
||||
if woken != 1 || !slices.Equal(dropped, []string{runGroupPrefix + "r1"}) {
|
||||
t.Errorf("woken %d (want 1), dropped %v (want the run's candidate)", woken, dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at
|
||||
// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before.
|
||||
func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
|
||||
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
|
||||
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
|
||||
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
|
||||
// says so in the journal; the same end said again naming the version writes it. H-A-490: a version released before its pipeline
|
||||
// passed (an again pipeline on a built, published key) gets its want at the pass, matched to the mover's build by its parts.
|
||||
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
follow bool
|
||||
passed bool
|
||||
want bool
|
||||
}{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} {
|
||||
name string
|
||||
passed, releasedBefore bool
|
||||
}{{"no pipeline passed", false, false}, {"passed, then its version released", true, false}, {"released, then its pipeline passed", true, true}} {
|
||||
passed := c.passed
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
|
||||
endDone(t, p.w, "web", fp, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey("web", fp)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
p.w.put(buckets.Have, "g14.web", have)
|
||||
composes(t, p.w, "g14", digest)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web", Requires: []string{"database"}}}
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
on := true
|
||||
p.w.heardPresence("g14", !on, time.Now())
|
||||
if c.follow {
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
p.ends.on.Store(true)
|
||||
if c.passed {
|
||||
_ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true)
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
var journal []string
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
p.ends.trunk = p.w.trunkName
|
||||
p.ends.releasedOf = p.w.releasedOf
|
||||
if c.releasedBefore {
|
||||
// Built and released in an earlier pipeline that failed: the runner's again pipeline ends naming no version.
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 4, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "web:image", Digest: "sha256:7"}}})
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: fp, Outcome: builds.BuildDone, Made: map[string]string{"image": "sha256:7"}})
|
||||
if err := p.w.released.put(rel); err != nil || p.w.ends.put(built) != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
} else if passed {
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
p.wants(context.Background())
|
||||
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
|
||||
t.Errorf("an end naming no version wrote %d wants and said %q", len(wantsKV.values), journal)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 2, mpr.StepResultOutcomePassed)))
|
||||
}
|
||||
p.wants(context.Background())
|
||||
key, _ := mwc.WantKey("g14", "web")
|
||||
if _, got := wantsKV.values[key]; got != c.want {
|
||||
t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
if c.follow && c.passed {
|
||||
raw, _, _ := p.ends.st.get(endKey("web", trunkBranch))
|
||||
var end End
|
||||
_ = json.Unmarshal(raw, &end)
|
||||
if !end.Written {
|
||||
t.Errorf("%s: the end was not marked written", c.name)
|
||||
for _, node := range []string{"g14", "ace"} {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
raw, got := wantsKV.values[key]
|
||||
if got != passed {
|
||||
t.Errorf("%s: a want on %s written = %v", c.name, node, got)
|
||||
continue
|
||||
}
|
||||
if !passed {
|
||||
continue
|
||||
}
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
|
||||
t.Errorf("%s: the want on %s = %+v (%v)", c.name, node, w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and
|
||||
// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default.
|
||||
func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) {
|
||||
// Design 47 G §353, §352: a registration's first branch is a placement's default, and a branch it no longer watches loses its
|
||||
// end; a registration gone loses them all.
|
||||
func TestARegistrationNamesTheDefaultBranch(t *testing.T) {
|
||||
st := newMemEnds()
|
||||
woken := 0
|
||||
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
|
||||
e.trunk = func() string { return trunkBranch }
|
||||
_ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false)
|
||||
_ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true)
|
||||
if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) {
|
||||
t.Fatal("the follower held a want before any registration")
|
||||
}
|
||||
pass := mpr.PipelineEndedOutcomePassed
|
||||
_, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed)))
|
||||
_, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed)))
|
||||
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
|
||||
if !e.following() || woken == 0 {
|
||||
t.Error("the first registration did not switch the wants on")
|
||||
endsKept := func() (n int) { // a pipeline's tier progress is not a module's end
|
||||
for k := range st.vals {
|
||||
if !isTierKey(k) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) {
|
||||
t.Error("after the switch the wants did not follow the ends")
|
||||
if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
|
||||
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", ""))
|
||||
}
|
||||
if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
|
||||
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", ""))
|
||||
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" {
|
||||
t.Errorf("the default branch's target = %+v %v", tg, ok)
|
||||
}
|
||||
e.registered(mpcat.Registration{Module: "web"}, true)
|
||||
if len(st.vals) != 0 {
|
||||
t.Errorf("a registration gone left %d ends", len(st.vals))
|
||||
if endsKept() != 0 {
|
||||
t.Errorf("a registration gone left %d ends", endsKept())
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a
|
||||
// change on `beta` moves only the want of the node whose placement chose `beta`.
|
||||
func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
// Design 47 G §14, §184 (test 68): a node's want follows the branch its placement names, so a version passed on `beta` moves only
|
||||
// the want of the node whose placement chose `beta`.
|
||||
func TestAVersionOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
@@ -311,15 +254,8 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
on := true
|
||||
for _, node := range []string{"g14", "ace"} {
|
||||
composes(t, p.w, node, digest)
|
||||
p.w.heardPresence(node, !on, time.Now())
|
||||
}
|
||||
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
|
||||
pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}
|
||||
values, err := mmac.AssignmentsValues(pl)
|
||||
values, err := mmac.AssignmentsValues(mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -327,24 +263,10 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
p.w.put(plc.AssignmentsBucket(), key, raw)
|
||||
}
|
||||
}
|
||||
// publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline.
|
||||
publish := func(branch, f string, source int64) {
|
||||
mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source)
|
||||
if branch == "beta" {
|
||||
mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source)
|
||||
}
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, branch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
|
||||
endDone(t, p.w, "web", f, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey("web", f)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
_ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true)
|
||||
}
|
||||
run := func(node, f string) {
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
p.w.put(buckets.Have, node+".web", have)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp("a"), Manifest: strings.Repeat("e", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, "main", "web", mod)
|
||||
passed := func(branch, f, version string, source int64) {
|
||||
_, _ = p.ends.take(ended(branch, source, mpr.PipelineEndedOutcomePassed, mover("web", f, version, 1, mpr.StepResultOutcomePassed)))
|
||||
}
|
||||
fpWant := func(node string) string {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
@@ -354,18 +276,16 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
publish("main", fp("a"), 5)
|
||||
publish("beta", fp("b"), 5)
|
||||
run("g14", fp("a"))
|
||||
run("ace", fp("b"))
|
||||
passed("main", fp("a"), "0.1.1", 5)
|
||||
passed("beta", fp("b"), "0.2.1", 5)
|
||||
p.wants(context.Background())
|
||||
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
|
||||
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
|
||||
}
|
||||
publish("beta", fp("d"), 6)
|
||||
passed("beta", fp("d"), "0.2.2", 6)
|
||||
p.wants(context.Background())
|
||||
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
|
||||
t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
|
||||
t.Errorf("a version on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// placementBy is who a put the placement's build makes names as its asker: the placement, whose proposal the operator
|
||||
// approved, is the warrant (design 47 *Going back is one act*; ADR 0666 §24).
|
||||
const placementBy = "the placement"
|
||||
|
||||
// placementPuts makes each placed module's `build` a held put of that build, and releases a put the placement made once its
|
||||
// placement names that build no more (design 47 line 912: "a return to an earlier build is one value in the placement, the
|
||||
// module's `build` in its placement key, set by a placement proposal"; ADR 0666 §24; B-K1). The put then goes its own steps
|
||||
// (the builder's build, the composition, an ask where the module's policy needs one, the want, holding) and is read with
|
||||
// `assigner_puts` and released with `assigner_release` like any other; a placement's build the builder never built is refused
|
||||
// by its build step, said in the put's words. A put asked by a person stands until the placement names another build.
|
||||
func (p *passer) placementPuts(ctx context.Context) {
|
||||
kv := p.kvs[buckets.Puts()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
now := p.clock().UTC()
|
||||
named := map[string]string{} // put key -> the build its placement names
|
||||
for node, a := range p.w.assignments() {
|
||||
for _, m := range a.Modules {
|
||||
if m.Build == "" {
|
||||
continue
|
||||
}
|
||||
if key, err := puts.Key(node, m.Module); err == nil {
|
||||
named[key] = m.Build
|
||||
}
|
||||
}
|
||||
}
|
||||
held := p.w.read(buckets.Puts())
|
||||
keys := make([]string, 0, len(named)+len(held))
|
||||
for k := range named {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
for k := range held {
|
||||
if _, there := named[k]; !there {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
build, names := named[key]
|
||||
var before *puts.Put
|
||||
if raw, ok := held[key]; ok {
|
||||
if got, err := puts.Decode(raw); err == nil {
|
||||
before = &got
|
||||
}
|
||||
}
|
||||
var next puts.Put
|
||||
switch {
|
||||
case names && before != nil && before.FP == build:
|
||||
continue // made already, or a person's put of the same build: it goes its own steps, ended or not
|
||||
case names:
|
||||
words := fmt.Sprintf("asked by %s: it names build %s of %s", placementBy, shortFP(build), key)
|
||||
if before != nil && !puts.Ended(before.Step) {
|
||||
words += fmt.Sprintf("; it replaces the put of %s, which stood at %s", shortFP(before.FP), before.Step)
|
||||
}
|
||||
next = puts.Put{FP: build, Hold: true, Why: "the placement names this build (ADR 0666 §24)", By: placementBy, At: now}.Said(now, puts.Asked, words)
|
||||
case before != nil && before.By == placementBy && !puts.Ended(before.Step):
|
||||
next = before.Said(now, puts.Released, fmt.Sprintf("released by %s: it names no build of its own any more; the assigner's rules write the want again", placementBy))
|
||||
default:
|
||||
continue
|
||||
}
|
||||
data, err := puts.Encode(next)
|
||||
if err != nil {
|
||||
p.logf("the placement's put %s is not written: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if before == nil {
|
||||
_, err = kv.Create(ctx, key, data)
|
||||
} else if e, gerr := kv.Get(ctx, key); gerr != nil {
|
||||
err = gerr
|
||||
} else {
|
||||
_, err = kv.Update(ctx, key, data, e.Revision())
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the placement's put %s could not be written, and is written at the next pass: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(buckets.Puts(), key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
if p.acts != nil {
|
||||
p.acts.say(key, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
|
||||
func orderFP(c string) string {
|
||||
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
|
||||
}
|
||||
|
||||
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
|
||||
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
|
||||
type orderMod struct {
|
||||
name string
|
||||
run, trunk string
|
||||
forms map[string]int
|
||||
holds []string
|
||||
}
|
||||
|
||||
type orderFixture struct {
|
||||
t *testing.T
|
||||
p *passer
|
||||
wants *memKV
|
||||
events []string
|
||||
bodies []meshwants.Wait
|
||||
answers chan struct{}
|
||||
}
|
||||
|
||||
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
|
||||
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
|
||||
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
|
||||
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
|
||||
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p := f.p
|
||||
p.logf = t.Logf
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
f.events = append(f.events, event)
|
||||
f.bodies = append(f.bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
pl := mma.Assignments{Node: "ace"}
|
||||
forms := map[string]int{} // by fingerprint: each module's own
|
||||
modules := []string{}
|
||||
for _, m := range mods {
|
||||
modules = append(modules, m.name)
|
||||
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
|
||||
for _, seat := range m.holds {
|
||||
scope := "node"
|
||||
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
|
||||
scope = "mesh"
|
||||
}
|
||||
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
|
||||
}
|
||||
for fp, form := range m.forms {
|
||||
forms[fp] = form
|
||||
}
|
||||
mdigest := fmt.Sprintf("%064x", len(modules))
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
endDone(t, p.w, m.name, m.trunk, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
f.runs(m.name, m.run)
|
||||
// The trunk's build is released with a passed pipeline (design 47 H §750).
|
||||
_, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover(m.name, m.trunk, "0.1.1", 1, mpr.StepResultOutcomePassed)))
|
||||
wkey, _ := mwc.WantKey("ace", m.name)
|
||||
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
|
||||
p.w.put(buckets.Wants(), wkey, old)
|
||||
f.wants.values[wkey] = old
|
||||
}
|
||||
composes(t, p.w, "ace", digest)
|
||||
values, err := mmac.AssignmentsValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(plc.AssignmentsBucket(), key, raw)
|
||||
}
|
||||
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
|
||||
objects := memObjects{}
|
||||
for i, at := range history {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, module := range modules {
|
||||
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
|
||||
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
|
||||
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
|
||||
}
|
||||
placed := mma.Assignments{Node: "ace"}
|
||||
for _, module := range modules {
|
||||
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
|
||||
}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]mma.Assignments{"ace": placed}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
|
||||
if form := forms[in.Fingerprint]; form > 0 {
|
||||
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
|
||||
return f
|
||||
}
|
||||
|
||||
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
|
||||
func (f *orderFixture) runs(module, fp string) {
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
f.p.w.put(buckets.Have, "ace."+module, have)
|
||||
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
|
||||
}
|
||||
|
||||
// pass makes a pass, and after it every catalog entry it asked is answered.
|
||||
func (f *orderFixture) pass() {
|
||||
f.t.Helper()
|
||||
f.p.wants(context.Background())
|
||||
asked := 0
|
||||
for _, row := range f.p.entries.Rows() {
|
||||
if row.State == meshcatalogclient.EntryAsked {
|
||||
asked++
|
||||
}
|
||||
}
|
||||
for range asked {
|
||||
select {
|
||||
case <-f.answers:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
f.t.Fatal("an entry asked was not answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wantAt is the fingerprint module is wanted at on ace.
|
||||
func (f *orderFixture) wantAt(module string) string {
|
||||
f.t.Helper()
|
||||
key, _ := mwc.WantKey("ace", module)
|
||||
w, err := mwc.DecodeWant(f.wants.values[key])
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
|
||||
func (f *orderFixture) waits() []string {
|
||||
var out []string
|
||||
for key := range f.p.w.read(buckets.Waits()) {
|
||||
out = append(out, key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A3/AC1 (design 47 H §771, §764): given a module whose kept data's form changes while the new versions of the node-backup holder and
|
||||
// of the bus module do not run, when the assigner passes, then the module is wanted at its new version, no `order.` key is kept and no
|
||||
// wait is said: the order on a machine is its node-engine's apply order, by rank.
|
||||
func TestNoWantWaitsForTheOrderOfADelivery(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
|
||||
})
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 0 {
|
||||
t.Fatalf("web at %s, waits %v, events %v: want web at its new version, no wait kept or said", got, f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -42,10 +43,10 @@ type caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
|
||||
CallKept(key string, body any) (json.RawMessage, error)
|
||||
// CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668).
|
||||
CallKeptOn(node, key string, body any) (json.RawMessage, error)
|
||||
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
|
||||
CallFile(key string, body any) (json.RawMessage, error)
|
||||
// CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775).
|
||||
CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
@@ -106,7 +107,6 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if p.acts == nil {
|
||||
return
|
||||
}
|
||||
p.placementPuts(ctx)
|
||||
all := p.putsByModule()
|
||||
defer func() { p.putWaits(ctx, p.putsByModule()) }()
|
||||
if len(all) == 0 {
|
||||
@@ -136,14 +136,6 @@ func (p *passer) puts(ctx context.Context) {
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) {
|
||||
if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
@@ -189,6 +181,7 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
|
||||
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
|
||||
w.Build, w.BuildWords = e.Outcome, e.Error
|
||||
}
|
||||
w.Version, _ = p.w.releasedOf(module, put.FP) // matched by its build's parts, as the pass matches a mover's (H-A-490 addendum)
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
|
||||
type releaseHeard struct {
|
||||
Version string
|
||||
Parts map[string]string
|
||||
Source int64
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
|
||||
func presenceOf(data []byte) (string, time.Time, error) {
|
||||
var e wd.MachineAwayEvent // machine-back has the same fields
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
if e.Machine == "" {
|
||||
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
|
||||
}
|
||||
return e.Machine, e.At, nil
|
||||
}
|
||||
|
||||
// hearReleases reads whole, from their first message, then follows, the catalog seat's version-released and the watchman's
|
||||
// machine-away and machine-back (mesh-sdk process.FromFirst, A-233), never their state. A stream not heard is said as the assigner's finding, and what it carries is not known until
|
||||
// the assigner starts again.
|
||||
func hearReleases(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
|
||||
acct := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
nh := notHeardWriter(findings, logf)
|
||||
for _, s := range []struct {
|
||||
stream, words string
|
||||
take func([]byte) error
|
||||
}{
|
||||
{meshcatalog.StreamVersionReleased, "no want names a released version", w.released.put},
|
||||
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, true, at)
|
||||
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", m)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, false, at)
|
||||
logf("%s is back (machine-back): a composition step composes it again", m)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
take := s.take
|
||||
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
|
||||
if err := take(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf, process.FromFirst())
|
||||
if err != nil {
|
||||
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
|
||||
continue
|
||||
}
|
||||
nh.Heard(s.stream)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -133,4 +133,22 @@ func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
|
||||
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
|
||||
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
|
||||
}
|
||||
|
||||
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
|
||||
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
|
||||
for _, withWait := range []bool{true, false} {
|
||||
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
|
||||
if withWait {
|
||||
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
|
||||
}
|
||||
raw, err := nodeengineclient.EncodeHave(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Have, hkey, raw)
|
||||
pass()
|
||||
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
|
||||
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// The assignment graph revisions are removed when they are no longer current (design 47 H §764, H-A-86): the gate's pins of
|
||||
// them went with its pinned, freed, heads-replaced and pins-stated (H-A-48), and with them the assigner's state pins and its
|
||||
// tools assigner_pins and assigner_unpin. A revision that is not current (the newest the assigner wrote) is removed, its object
|
||||
// and the entry objects no kept revision or candidate lists and no want or running build names (G §179), when a revision ceases
|
||||
// to be current and once at the start. Never on a timer.
|
||||
|
||||
// revisionObjects is the object bucket of the revisions as the remover removes from it.
|
||||
type revisionObjects interface {
|
||||
assigngraph.Objects
|
||||
Delete(ctx context.Context, name string) error
|
||||
}
|
||||
|
||||
// revisionRemover removes the revisions that are not current.
|
||||
type revisionRemover struct {
|
||||
ctx context.Context
|
||||
objects revisionObjects
|
||||
// inUse says a want or a running build names the module's fingerprint (§179).
|
||||
inUse func(module, fp string) bool
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// judge removes every revision that is not current; a removal that fails is said and made at the next judging.
|
||||
func (k *revisionRemover) judge() {
|
||||
if err := k.remove(); err != nil {
|
||||
k.logf("the revisions that are not current are not removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// trunkRevision is the revision an object of a trunk revision names, and whether it names one.
|
||||
func trunkRevision(name string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(name, "trunk.")
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
r, err := strconv.ParseInt(n, 10, 64)
|
||||
return r, err == nil && r > 0
|
||||
}
|
||||
|
||||
// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or
|
||||
// running build names (design 47 G §179).
|
||||
func (k *revisionRemover) remove() error {
|
||||
names, described, err := k.listing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var kept []int64
|
||||
for _, n := range names {
|
||||
if r, ok := trunkRevision(n); ok {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil
|
||||
}
|
||||
current := slices.Max(kept)
|
||||
listed := map[string]bool{}
|
||||
removed := 0
|
||||
for _, r := range kept {
|
||||
if r == current {
|
||||
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
|
||||
return fmt.Errorf("revision %d: %w", r, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
if removed == 0 {
|
||||
return nil
|
||||
}
|
||||
// A run's candidate kept now lists its entries too, until its run's pipeline-ended drops it (A-201, A-235).
|
||||
for _, n := range names {
|
||||
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
}
|
||||
for _, n := range names {
|
||||
digest, ok := strings.CutPrefix(n, "entry.")
|
||||
if !ok || listed[digest] {
|
||||
continue
|
||||
}
|
||||
e, err := k.entryOf(n, digest, described)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, n); err != nil {
|
||||
return fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
}
|
||||
k.logf("%d assignment graph revisions not current are removed", removed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
|
||||
func inUseOf(w *world) func(module, fp string) bool {
|
||||
return func(module, fp string) bool {
|
||||
for _, want := range w.wantsOf()[module] {
|
||||
if want.Fp == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, mods := range w.runningNow() {
|
||||
if r, ok := mods[module]; ok && r.Fingerprint == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// listing is every object's name in one read and, where the bucket says them, each entry object's module and fingerprint by name
|
||||
// (design 47 H-A-259); nil described where the bucket says none.
|
||||
func (k *revisionRemover) listing() ([]string, map[string]assigngraph.Described, error) {
|
||||
d, ok := k.objects.(assigngraph.Describer)
|
||||
if !ok {
|
||||
names, err := k.objects.Names(k.ctx)
|
||||
return names, nil, err
|
||||
}
|
||||
listed, err := d.Described(k.ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
names := make([]string, 0, len(listed))
|
||||
described := make(map[string]assigngraph.Described, len(listed))
|
||||
for _, l := range listed {
|
||||
names = append(names, l.Name)
|
||||
described[l.Name] = l
|
||||
}
|
||||
return names, described, nil
|
||||
}
|
||||
|
||||
// entryOf is the module and fingerprint of the entry object name: as the listing says them, with no read; an entry put before the
|
||||
// listing said them is read once and put again with them, so no later judging reads it (design 47 H-A-259).
|
||||
func (k *revisionRemover) entryOf(name, digest string, described map[string]assigngraph.Described) (meshwants.AssignmentEntry, error) {
|
||||
if l, ok := described[name]; ok && l.Module != "" && l.Fp != "" {
|
||||
return meshwants.AssignmentEntry{Module: l.Module, Fp: l.Fp}, nil
|
||||
}
|
||||
data, err := k.objects.Get(k.ctx, name)
|
||||
if err != nil {
|
||||
return meshwants.AssignmentEntry{}, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
var e meshwants.AssignmentEntry
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return e, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
if described != nil {
|
||||
if err := assigngraph.PutEntry(k.ctx, k.objects, name, data, e); err != nil {
|
||||
k.logf("the entry %s is not put again with its module and fingerprint, so the next judging reads it again: %v", digest, err)
|
||||
}
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
|
||||
|
||||
// describedObjects is memObjects whose listing says each entry's module and fingerprint (assigngraph.Describer), counting the
|
||||
// entry objects read.
|
||||
type describedObjects struct {
|
||||
memObjects
|
||||
meta map[string][2]string
|
||||
reads int
|
||||
}
|
||||
|
||||
func (d *describedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if strings.HasPrefix(name, "entry.") {
|
||||
d.reads++
|
||||
}
|
||||
return d.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
func (d *describedObjects) PutDescribed(_ context.Context, name string, data []byte, module, fp string) error {
|
||||
d.memObjects[name], d.meta[name] = data, [2]string{module, fp}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *describedObjects) Described(context.Context) ([]assigngraph.Described, error) {
|
||||
var out []assigngraph.Described
|
||||
for n := range d.memObjects {
|
||||
out = append(out, assigngraph.Described{Name: n, Module: d.meta[n][0], Fp: d.meta[n][1]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the
|
||||
// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists,
|
||||
// a running build's entry and a run's kept candidate with its entries stay.
|
||||
// H-A-259: where the listing says each entry's module and fingerprint, the judging reads no entry object; an entry put before it
|
||||
// said them is read once and put again with them.
|
||||
func TestARevisionNotCurrentIsRemoved(t *testing.T) {
|
||||
for _, listed := range []string{"plain", "described", "described-later"} {
|
||||
objects := memObjects{}
|
||||
described := &describedObjects{memObjects: objects, meta: map[string][2]string{}}
|
||||
entry := func(module, fp string) string {
|
||||
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
|
||||
d := assigngraph.EntryDigest(e)
|
||||
raw, _ := json.Marshal(e)
|
||||
objects[assigngraph.EntryObject(d)] = raw
|
||||
if listed == "described" {
|
||||
described.meta[assigngraph.EntryObject(d)] = [2]string{module, fp}
|
||||
}
|
||||
return d
|
||||
}
|
||||
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
|
||||
revision := func(n int64, digests ...string) {
|
||||
rev := meshwants.AssignmentRevision{Revision: n}
|
||||
for _, d := range digests {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
|
||||
}
|
||||
raw, _ := json.Marshal(rev)
|
||||
objects[assigngraph.TrunkObject(n)] = raw
|
||||
}
|
||||
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
|
||||
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
|
||||
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
|
||||
revision(1, shared, old, running)
|
||||
revision(2, shared, old)
|
||||
revision(3, shared)
|
||||
var o revisionObjects = objects
|
||||
if listed != "plain" {
|
||||
o = described
|
||||
}
|
||||
k := &revisionRemover{ctx: context.Background(), objects: o, logf: t.Logf,
|
||||
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
|
||||
k.judge()
|
||||
has := func(name string) bool { _, ok := objects[name]; return ok }
|
||||
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
|
||||
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
|
||||
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
|
||||
if has(name) != want {
|
||||
t.Errorf("%s: %s kept = %v, want %v", listed, name, has(name), want)
|
||||
}
|
||||
}
|
||||
switch listed {
|
||||
case "described":
|
||||
if described.reads != 0 {
|
||||
t.Errorf("described: the judging read %d entry objects, want none", described.reads)
|
||||
}
|
||||
case "described-later":
|
||||
if described.meta[assigngraph.EntryObject(running)] != [2]string{"c", "fp-c1"} {
|
||||
t.Errorf("described-later: the entry in use is not put again with its module and fingerprint: %v", described.meta)
|
||||
}
|
||||
reads := described.reads
|
||||
revision(4, shared)
|
||||
k.judge()
|
||||
if described.reads != reads {
|
||||
t.Errorf("described-later: a second judging read %d entry objects again", described.reads-reads)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
mdcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -266,6 +265,7 @@ func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string)
|
||||
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = mmac.ProposalAsked, placed.ID
|
||||
p.logf("the proposal %s is asked of the operator: its ask %s is written under %s, which the teller raises on the operator channel", prop.ID, placed.ID, akey)
|
||||
return prop, true
|
||||
}
|
||||
|
||||
@@ -340,12 +340,20 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case mdcc.VerdictFail:
|
||||
case plc.VerdictFail:
|
||||
prop.State, prop.Ask = mmac.ProposalFailed, ""
|
||||
case plc.VerdictWaits:
|
||||
prop.State, prop.Ask = mmac.ProposalAnswering, ""
|
||||
case mdcc.VerdictPass:
|
||||
case plc.VerdictPass:
|
||||
if prop.State != mmac.ProposalAsked || again {
|
||||
// A proposal none of whose changes the setting ask-operator-for names is applied at once, with no ask (A-425a).
|
||||
if kinds := proposalKinds(prop, v); !p.asksFor(kinds) {
|
||||
words := fmt.Sprintf("applied with no ask: its changes (%s) are none of the kinds the setting ask-operator-for names (%s); proposed by %s: %s",
|
||||
strings.Join(kinds, ", "), strings.Join(p.askOperatorFor(), ", "), prop.By, prop.Why)
|
||||
p.logf("the proposal %s is %s", prop.ID, words)
|
||||
p.apply(ctx, prop, rev, mdc.Answer{Words: words}, current, revs)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
@@ -499,6 +507,12 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
p.apply(ctx, prop, rev, answer, current, revs)
|
||||
}
|
||||
|
||||
// apply writes a proposal's laid candidates, its answer carried by assignments-changed: the operator's warranted answer, or the
|
||||
// words of a proposal applied with no ask (design 47 G A-425a).
|
||||
func (p *passer) apply(ctx context.Context, prop mma.Proposal, rev uint64, answer mdc.Answer, current map[string]mma.Assignments, revs map[string]uint64) {
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -67,13 +68,17 @@ func indexerView(ask func(string, any) (json.RawMessage, error), state string) (
|
||||
}
|
||||
page := views.AskVia(ask, tool)
|
||||
out := map[string]json.RawMessage{}
|
||||
started, pages := time.Now(), 0
|
||||
for after := ""; ; {
|
||||
a, err := page(context.Background(), after)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err)
|
||||
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
|
||||
}
|
||||
pages++
|
||||
maps.Copy(out, a.Values)
|
||||
if a.Next == "" || a.Next == after {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond))
|
||||
return out, nil
|
||||
}
|
||||
after = a.Next
|
||||
@@ -88,6 +93,11 @@ type seatTools struct {
|
||||
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
|
||||
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
|
||||
seatList *seatlist.Part
|
||||
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
|
||||
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
|
||||
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
|
||||
// direct listing.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
@@ -103,25 +113,18 @@ type keyed struct {
|
||||
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
|
||||
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(mma.StateAssignments)
|
||||
all, err := readAll(s.st, mma.StateAssignments)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("assignments cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("assignments cannot be read: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys := slices.Sorted(maps.Keys(all))
|
||||
held := mmac.NewKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := mmac.ParseKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(mma.StateAssignments, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("assignments key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
if _, err := held.Put(key, all[key].raw, all[key].rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
@@ -145,20 +148,18 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
haves, err := readAll(s.st, haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be read: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHaveNode(v.raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
@@ -239,11 +240,21 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
started := time.Now()
|
||||
step := func(what string, err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s failed after %s: %v\n", by, what, time.Since(started).Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s at %s\n", by, what, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
current, revs, err := s.assignments()
|
||||
step("assignments read", err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
step("the indexer's modules and the nodes' records read", err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -259,11 +270,15 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
step(fmt.Sprintf("refused (%d findings)", len(findings)), nil)
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
step("judged", nil)
|
||||
if err := s.write(p); err != nil {
|
||||
step("the proposal "+p.ID+" written", err)
|
||||
return nil, err
|
||||
}
|
||||
step("the proposal "+p.ID+" written", nil)
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
@@ -297,10 +312,10 @@ func warrantOf(v any) (*mma.Warrant, error) {
|
||||
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
|
||||
// `person.<module>`), else the caller as proven, which the refusal names.
|
||||
func moduleOfCaller(by string, enrollers []string) string {
|
||||
for _, module := range enrollers {
|
||||
if (stdio.Callers{module}).Allows(by) {
|
||||
return module
|
||||
}
|
||||
// The enrolment's caller stays (design 47 H-A-41): a module's bus user is `<node>.<module>`, a person's `person.<name>`.
|
||||
_, module, ok := strings.Cut(by, ".")
|
||||
if ok && !strings.HasPrefix(by, "person.") && slices.Contains(enrollers, module) {
|
||||
return module
|
||||
}
|
||||
return by
|
||||
}
|
||||
@@ -477,21 +492,17 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
haves, err := readAll(s.st, haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be read: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := nodeengineclient.DecodeHaveNode(raw)
|
||||
h, err := nodeengineclient.DecodeHaveNode(v.raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -548,25 +559,43 @@ func (s *seatTools) tools() []stdio.Tool {
|
||||
return out
|
||||
}
|
||||
|
||||
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
|
||||
// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
|
||||
func (s *seatTools) seatHolders() (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
|
||||
return vals, nil
|
||||
}
|
||||
}
|
||||
keys, err := s.st.keys(mma.StateSeatHolders)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []mma.SeatHolders{}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
if found {
|
||||
out[key] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
raws, err := s.seatHolders()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(raws))
|
||||
out := []mma.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
var sh mma.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
if json.Unmarshal(raws[key], &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
|
||||
|
||||
@@ -20,11 +20,9 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
@@ -270,19 +268,6 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
})
|
||||
}
|
||||
s.seatList = nil
|
||||
t.Run("1966: the seat list's again tool answers an operator's session through the mesh MCP server", func(t *testing.T) {
|
||||
// Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. The list names
|
||||
// the seats, resolved from the assigner's own seat-holders (A-295, A-298): a module holding neither is refused.
|
||||
held := map[string][]byte{"node-tool-runner.shanks": []byte(`{"holders":[{"module":"node-tools","node":"shanks"}]}`),
|
||||
"mesh-healers": []byte(`{"holders":[{"module":"mesh-delivery-watchman","node":"ace"}]}`)}
|
||||
holders := seatlist.HoldersIn(func() (map[string][]byte, error) { return held, nil })
|
||||
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-delivery-watchman": false,
|
||||
"ace.mesh-healers": true, "ace.messenger": true} {
|
||||
if err := seatlist.RefusalWith("assigner_seat_list_again", seatListAgainCallers, caller, holders); (err != nil) != refused {
|
||||
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
@@ -325,11 +310,11 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
place("ace", mma.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
place("g14", mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
ctx := context.Background()
|
||||
if p.seatHolders(ctx); writes != 0 {
|
||||
if p.seatHolders(ctx, nil); writes != 0 {
|
||||
t.Fatalf("seat-holders was written from an unread seat list: %v", kv.values)
|
||||
}
|
||||
p.seats = gateOver(t, false)
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
var broker mma.SeatHolders
|
||||
if err := json.Unmarshal(kv.values["mesh-broker"], &broker); err != nil || broker.Outside {
|
||||
t.Errorf("mesh-broker's holders %+v, %v: a seat the list names is no seat outside it", broker, err)
|
||||
@@ -337,12 +322,12 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
|
||||
t.Fatalf("%d writes: %v", writes, kv.values)
|
||||
}
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
if writes != 3 {
|
||||
t.Errorf("an unchanged assignments wrote %d keys", writes-3)
|
||||
}
|
||||
place("g14") // g14 holds nothing now
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
|
||||
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
|
||||
}
|
||||
@@ -352,6 +337,53 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
t.Errorf("seats on ace = %+v, %v", out, err)
|
||||
}
|
||||
})
|
||||
t.Run("issue 1982: assignments with seats reads the tools process's watched copy of seat-holders, the store only while it cannot", func(t *testing.T) {
|
||||
seeded := map[string][]byte{}
|
||||
put := func(seat, node string, holders ...mma.SeatHoldersHoldersItem) {
|
||||
raw, err := json.Marshal(mma.SeatHolders{Seat: seat, Node: node, Holders: holders})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seeded[seat] = raw
|
||||
}
|
||||
put("mesh-broker", "", mma.SeatHoldersHoldersItem{Module: "nats", Node: "ace"})
|
||||
put("node-shell.g14", "g14")
|
||||
st := &countingSeatStore{memStore: memStore{mma.StateSeatHolders: &memKV{values: seeded}}}
|
||||
sv := newSeatTools(st, nil)
|
||||
watches := 0
|
||||
sv.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
watches++
|
||||
for k, v := range seeded {
|
||||
if err := handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, func(state, key string) ([]byte, bool, error) {
|
||||
v, ok := seeded[key]
|
||||
return v, ok, nil
|
||||
})
|
||||
out, err := sv.seats("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || watches != 1 || st.keysCalls != 0 || st.getCalls != 0 {
|
||||
t.Errorf("seats from the watched copy = %+v, watched %d time(s), the store's own keys %d get %d (want none)",
|
||||
got, watches, st.keysCalls, st.getCalls)
|
||||
}
|
||||
// The lead's answer on issue 1982: the verb falls back to the store's own listing only while the watched copy
|
||||
// cannot be read (here, its watch refused); the answer's form does not change.
|
||||
sv.watched = stdio.WatchedStatesOver(func(string, func(stdio.StateChange) error) error {
|
||||
return errors.New("refused")
|
||||
}, func(state, key string) ([]byte, bool, error) { return nil, false, nil })
|
||||
out, err = sv.seats("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || st.keysCalls != 1 || st.getCalls != 2 {
|
||||
t.Errorf("seats falling back to the store = %+v, the store's own keys %d get %d (want 1, 2)", got, st.keysCalls, st.getCalls)
|
||||
}
|
||||
})
|
||||
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
@@ -486,51 +518,14 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
})
|
||||
}
|
||||
})
|
||||
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{
|
||||
Put: putInto(findings),
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
|
||||
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"})
|
||||
p.w.putEntry(buckets.Provisions, "ace.photos.db", failed, 1, time.Now())
|
||||
p.kvs[buckets.Provisions] = &memKV{}
|
||||
of := p.provisionsOf()
|
||||
if got := of("photos")("ace"); got["ace.photos.db"].State != meshprovisionerclient.Failed {
|
||||
t.Fatalf("provisions = %+v", got)
|
||||
}
|
||||
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
|
||||
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Provided})
|
||||
p.w.putEntry(buckets.Provisions, "ace.photos.db", provided, 2, time.Now())
|
||||
p.provisionsOf()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings after provided = %v", findings)
|
||||
}
|
||||
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
|
||||
ctx := context.Background()
|
||||
p.kvs[plc.PulsesBucket()] = &memKV{values: map[string][]byte{}}
|
||||
gone := map[string]bool{"ace.photos.s3": true}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
|
||||
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if len(findings) != 0 {
|
||||
t.Fatal("said before its bound")
|
||||
}
|
||||
p.w.put(plc.PulsesBucket(), "provision-missing.ace.photos.s3", nil) // the server expired it
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
|
||||
t.Fatalf("past its bound: findings %v", findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, map[string]bool{})
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("keyed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
// A whole state is read in pages where the store has them, never a key at a time (27 s for the assignments live,
|
||||
// 2026-11-14): the paged read answers what the key-by-key read answers, and asks no key alone.
|
||||
direct, err := readAll(st, haveState)
|
||||
paged := &pagedStore{memStore: st}
|
||||
viaPages, perr := readAll(paged, haveState)
|
||||
if err != nil || perr != nil || len(viaPages) != len(direct) || paged.pages == 0 || paged.gets != 0 {
|
||||
t.Errorf("readAll in pages = %d keys in %d pages with %d gets (%v, %v), want the %d keys of the direct read, no get", len(viaPages), paged.pages, paged.gets, err, perr, len(direct))
|
||||
}
|
||||
}
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
@@ -742,6 +737,23 @@ func readManifest() (catalogue.Manifest, error) {
|
||||
return catalogue.Decode(raw)
|
||||
}
|
||||
|
||||
// countingSeatStore is a memStore that counts its calls to keys and get, to prove a reader takes the state from the
|
||||
// watched copy and the store only while that copy cannot be read (issue 1982).
|
||||
type countingSeatStore struct {
|
||||
memStore
|
||||
keysCalls, getCalls int
|
||||
}
|
||||
|
||||
func (c *countingSeatStore) keys(state string) ([]string, error) {
|
||||
c.keysCalls++
|
||||
return c.memStore.keys(state)
|
||||
}
|
||||
|
||||
func (c *countingSeatStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
c.getCalls++
|
||||
return c.memStore.get(state, key)
|
||||
}
|
||||
|
||||
// countingKV is a memKV that counts its puts.
|
||||
type countingKV struct {
|
||||
memKV
|
||||
@@ -943,3 +955,35 @@ func (f *fakeIndexer) answer(key string, body any) (json.RawMessage, bool, error
|
||||
raw, err := json.Marshal(a)
|
||||
return raw, true, err
|
||||
}
|
||||
|
||||
// pagedStore is a memStore whose states are read in pages of one key, counting pages and single gets.
|
||||
type pagedStore struct {
|
||||
memStore
|
||||
pages, gets int
|
||||
}
|
||||
|
||||
func (p *pagedStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
p.gets++
|
||||
return p.memStore.get(state, key)
|
||||
}
|
||||
|
||||
func (p *pagedStore) page(state, after string) (stdio.StatePage, error) {
|
||||
p.pages++
|
||||
keys, err := p.memStore.keys(state)
|
||||
if err != nil {
|
||||
return stdio.StatePage{}, err
|
||||
}
|
||||
slices.Sort(keys)
|
||||
for i, k := range keys {
|
||||
if k <= after {
|
||||
continue
|
||||
}
|
||||
raw, rev, _, _ := p.memStore.get(state, k)
|
||||
pg := stdio.StatePage{Entries: []stdio.StateEntry{{Key: k, Value: raw, Revision: rev}}}
|
||||
if i < len(keys)-1 {
|
||||
pg.Next = k
|
||||
}
|
||||
return pg, nil
|
||||
}
|
||||
return stdio.StatePage{}, nil
|
||||
}
|
||||
|
||||
@@ -1,16 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
|
||||
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
|
||||
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
|
||||
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
|
||||
// (plc.CheckAssignments), and seat-holders is not written.
|
||||
|
||||
// A call through the mesh MCP server is proved only as its machine's tool runner (issue 627), so the holders of the seat
|
||||
// node-tool-runner stand for an operator's session until a session is proved (issue 1966). The list names seats, never a
|
||||
// module, resolved from the assigner's own seat-holders (design 47 G A-210, A-295, A-298).
|
||||
var seatListAgainCallers = []string{stdio.AnyPerson, stdio.CallerSeatPrefix + "node-tool-runner", stdio.CallerSeatPrefix + "mesh-healers"}
|
||||
|
||||
@@ -60,7 +60,7 @@ func gateOver(t *testing.T, silent bool) *seatlist.Part {
|
||||
// that waits for it to stay unread keeps short.
|
||||
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
|
||||
}
|
||||
g := seatlist.NewPart("assigner", seatListAgainCallers)
|
||||
g := seatlist.NewPart("assigner")
|
||||
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
|
||||
if r == nil {
|
||||
t.Fatal("the reader did not start")
|
||||
@@ -98,7 +98,7 @@ func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
|
||||
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
|
||||
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
|
||||
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
|
||||
if _, ok := seatlist.Definition(seatlist.NewPart("assigner", seatListAgainCallers).Reader, "mesh-wants")(); ok {
|
||||
if _, ok := seatlist.Definition(seatlist.NewPart("assigner").Reader, "mesh-wants")(); ok {
|
||||
t.Error("a definition is answered before the seat list's reader started")
|
||||
}
|
||||
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module a candidate adds with no release heard is said once, not each pass.
|
||||
func TestASkippedAddIsSaidOncePerModuleAndFingerprint(t *testing.T) {
|
||||
var logs []string
|
||||
p := &passer{logf: func(f string, a ...any) { logs = append(logs, strings.TrimSpace(f)) }}
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp2", "why")
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("said %d times, want 2: %v", len(logs), logs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2,
|
||||
// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's
|
||||
// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key
|
||||
// <seat>.<ask>, for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an
|
||||
// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask
|
||||
// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped.<ask> for
|
||||
// ask-taken-bound, and either ends its step could not run naming the drop and who asked it.
|
||||
|
||||
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
|
||||
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
|
||||
const (
|
||||
askTakenBoundSetting = "ask-taken-bound"
|
||||
defaultAskTakenBound = 10 * time.Minute
|
||||
)
|
||||
|
||||
// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404).
|
||||
const droppedPrefix = "dropped."
|
||||
|
||||
// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet.
|
||||
type takenStore interface {
|
||||
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
RenewTTL(key string, value any, ttl time.Duration) (uint64, error)
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg).
|
||||
type queueMsg interface {
|
||||
Data() []byte
|
||||
Headers() nats.Header
|
||||
Ack() error
|
||||
Nak() error
|
||||
Term() error
|
||||
InProgress() error
|
||||
}
|
||||
|
||||
// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end.
|
||||
type askDrop struct {
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
func (d *askDrop) Error() string {
|
||||
by := d.By
|
||||
if by == "" {
|
||||
by = "its asker"
|
||||
}
|
||||
return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why)
|
||||
}
|
||||
|
||||
// askIDOf is the ask's id a put carries: its event id, else its message id.
|
||||
func askIDOf(h nats.Header) string {
|
||||
if id := h.Get("x-event-id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return h.Get(nats.MsgIdHdr)
|
||||
}
|
||||
|
||||
// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is
|
||||
// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed
|
||||
// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken
|
||||
// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run
|
||||
// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice.
|
||||
func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) {
|
||||
id := askIDOf(m.Headers())
|
||||
var run meshwants.StepRun
|
||||
err := json.Unmarshal(m.Data(), &run)
|
||||
how := c.howOf(run.Step, false)
|
||||
if err != nil || id == "" || how == nil {
|
||||
c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if d := c.droppedBefore(id); d != nil {
|
||||
c.hold(id, m)
|
||||
c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d)
|
||||
c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "")
|
||||
if err := c.taken.Delete(droppedPrefix + id); err != nil {
|
||||
c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.running(id) {
|
||||
c.hold(id, m)
|
||||
c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
|
||||
return
|
||||
}
|
||||
first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound)
|
||||
if err != nil {
|
||||
c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
c.hold(id, m)
|
||||
if first {
|
||||
at := c.now().UTC()
|
||||
if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil {
|
||||
c.logf("saying ask-taken of %s: %v", id, err)
|
||||
}
|
||||
} else {
|
||||
c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline)
|
||||
}
|
||||
go c.say1(ctx, id, run, how, "")
|
||||
}
|
||||
|
||||
// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs.
|
||||
func (c *compositionSteps) hold(id string, m queueMsg) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.held == nil {
|
||||
c.held = map[string][]queueMsg{}
|
||||
}
|
||||
c.held[id] = append(c.held[id], m)
|
||||
}
|
||||
|
||||
// settle acknowledges (ack) or hands again (nak) every message held for id.
|
||||
func (c *compositionSteps) settle(id string, ack bool) {
|
||||
c.mu.Lock()
|
||||
ms := c.held[id]
|
||||
delete(c.held, id)
|
||||
c.mu.Unlock()
|
||||
for _, m := range ms {
|
||||
if ack {
|
||||
_ = m.Ack()
|
||||
} else {
|
||||
_ = m.Nak()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inProgressEvery says every message held for id in progress every d, until the returned stop is called.
|
||||
func (c *compositionSteps) inProgressEvery(id string) (stop func()) {
|
||||
every := time.Duration(0)
|
||||
if c.inProgress != nil {
|
||||
every = c.inProgress()
|
||||
}
|
||||
if every <= 0 {
|
||||
return func() {}
|
||||
}
|
||||
ticks := c.ticks
|
||||
if ticks == nil {
|
||||
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
|
||||
}
|
||||
tick, stopTicks := ticks(every)
|
||||
done, ended := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(ended)
|
||||
defer stopTicks()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick:
|
||||
c.mu.Lock()
|
||||
ms := append([]queueMsg(nil), c.held[id]...)
|
||||
c.mu.Unlock()
|
||||
for _, m := range ms {
|
||||
_ = m.InProgress()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { close(done); <-ended }
|
||||
}
|
||||
|
||||
// droppedBefore is the drop kept for an ask not pulled yet, nil for none.
|
||||
func (c *compositionSteps) droppedBefore(id string) *askDrop {
|
||||
if c.taken == nil {
|
||||
return nil
|
||||
}
|
||||
e, err := c.taken.Get(droppedPrefix + id)
|
||||
if err != nil {
|
||||
c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err)
|
||||
return nil
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var d askDrop
|
||||
if json.Unmarshal(e.Value, &d) != nil {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
|
||||
// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and
|
||||
// ends could not run naming it; one not running here is kept in taken as dropped.<ask> for ask-taken-bound, so its pull ends
|
||||
// it the same way. It answers whether the ask was held here.
|
||||
func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) {
|
||||
id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why)
|
||||
if id == "" || why == "" {
|
||||
return nil, errors.New("give the ask's id and why it is dropped")
|
||||
}
|
||||
by := strings.TrimSpace(in.By)
|
||||
if by == "" {
|
||||
by = caller
|
||||
}
|
||||
d := &askDrop{By: by, Why: why, At: c.now().UTC()}
|
||||
c.mu.Lock()
|
||||
stop := c.stops[id]
|
||||
c.mu.Unlock()
|
||||
if stop != nil {
|
||||
stop(d)
|
||||
c.logf("the ask %s running here is dropped: %v", id, d)
|
||||
return meshwants.DropAnswer{"held": true}, nil
|
||||
}
|
||||
if c.taken == nil {
|
||||
return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id)
|
||||
}
|
||||
if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil {
|
||||
return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err)
|
||||
}
|
||||
c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d)
|
||||
return meshwants.DropAnswer{"held": false}, nil
|
||||
}
|
||||
|
||||
// dropVerb is the process's handler of the seat mesh-wants' act drop.
|
||||
func dropVerb(c *compositionSteps) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.DropInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return c.dropAsk(caller, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
|
||||
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401).
|
||||
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
|
||||
return func() time.Duration {
|
||||
if d, ok := def(); ok {
|
||||
if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
|
||||
return time.Duration(worker.InProgressSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return ackWait() / 3
|
||||
}
|
||||
}
|
||||
|
||||
// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
|
||||
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
|
||||
func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error {
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)
|
||||
var once sync.Once
|
||||
bind := func() bool {
|
||||
cons, err := js.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
bound := false
|
||||
once.Do(func() {
|
||||
bound = true
|
||||
info := cons.CachedInfo()
|
||||
if info != nil && info.Config.AckWait > 0 {
|
||||
ackWait(info.Config.AckWait)
|
||||
}
|
||||
pending := 1000
|
||||
if info != nil && info.Config.MaxAckPending > 0 {
|
||||
pending = info.Config.MaxAckPending
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending))
|
||||
if err != nil {
|
||||
c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err)
|
||||
return
|
||||
}
|
||||
c.logf("the composition step is pulled from %s on %s", worker, stream)
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
})
|
||||
return bound
|
||||
}
|
||||
if bind() {
|
||||
return nil
|
||||
}
|
||||
c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream)
|
||||
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err)
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
|
||||
if bind() { // made between the first look and the subscription
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askTakenBound reads the setting ask-taken-bound (§781).
|
||||
func askTakenBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// kept is one key's value and revision, as a whole state read gives it.
|
||||
type kept struct {
|
||||
raw []byte
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// pager is a store that reads a state's values many to a request (stdio.KeptState.Page): the runtime's store.
|
||||
type pager interface {
|
||||
page(state, after string) (stdio.StatePage, error)
|
||||
}
|
||||
|
||||
// pageBytes and pageBudget bound one page of a whole state read, so its answer fits one bus message and its time.
|
||||
const (
|
||||
pageBytes = 768 * 1024
|
||||
pageBudget = 10 * time.Second
|
||||
)
|
||||
|
||||
func (runtimeStore) page(state, after string) (stdio.StatePage, error) {
|
||||
return stdio.State(state).Page(after, pageBytes, pageBudget)
|
||||
}
|
||||
|
||||
// readAll is every key of a state with its value and revision: in pages when the store has them, else key by key. Key by
|
||||
// key cost one bus request each, so the assignments (about 330 keys) took 27 s to read and a proposal outlasted its
|
||||
// caller's 30 s (2026-11-14), as the indexer's view of the trunk had the hour before.
|
||||
func readAll(st store, state string) (map[string]kept, error) {
|
||||
out := map[string]kept{}
|
||||
if p, ok := st.(pager); ok {
|
||||
for after := ""; ; {
|
||||
pg, err := p.page(state, after)
|
||||
if errors.Is(err, errors.ErrUnsupported) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", state, err)
|
||||
}
|
||||
for _, e := range pg.Entries {
|
||||
out[e.Key] = kept{raw: e.Value, rev: e.Revision}
|
||||
}
|
||||
if pg.Next == "" || pg.Next == after {
|
||||
return out, nil
|
||||
}
|
||||
after = pg.Next
|
||||
}
|
||||
}
|
||||
keys, err := st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
raw, rev, found, err := st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = kept{raw: raw, rev: rev}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// One pipeline's wants are written tier by tier (design 47 H §751, H-A-10, H-A-56): on a passed pipeline-ended the assigner
|
||||
// writes the wants of its first tier's movers; once each of them runs at the mover's fingerprint (the node-engine seat's
|
||||
// `running`, H-A-40) on every present placing machine, it writes the next tier's, and so on. The tier progress is the key
|
||||
// `pipeline.<branch>.<source>` of the state written, `{tier, since, movers}`; a tier counts placed movers only, and a machine away
|
||||
// is not waited for. A tier whose movers do not all run within `tier-bound` ends the delivery loudly: no later tier is written,
|
||||
// the finding `tier-late` (scope delivery, id the pipeline) is raised naming the movers and machines not running, and
|
||||
// `pipeline-wanted` says outcome tier-late; once every tier runs it says outcome running and the key goes (H-A-75). tier-late
|
||||
// clears when the late movers run or a later passed pipeline moves them (H-A-56).
|
||||
|
||||
// tierPrefix starts the key of a pipeline's tier progress in the state written.
|
||||
const tierPrefix = "pipeline."
|
||||
|
||||
// kindTierLate is the finding of a tier not running within tier-bound (design 47 H §751).
|
||||
const kindTierLate = "tier-late"
|
||||
|
||||
// defaultTierBound is tier-bound's default, 30 minutes (design 47 H §751).
|
||||
const defaultTierBound = 30 * time.Minute
|
||||
|
||||
// tierMover is one mover of a passed pipeline whose version the state written keeps: its module, fingerprint and tier.
|
||||
type tierMover struct {
|
||||
Module string `json:"module"`
|
||||
Fp string `json:"fp"`
|
||||
Tier int64 `json:"tier"`
|
||||
// NoVersion is a passed mover whose version is not yet written: its tier does not run until it is (H-A-277 (1)).
|
||||
NoVersion bool `json:"no-version,omitempty"`
|
||||
}
|
||||
|
||||
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source
|
||||
// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a
|
||||
// value kept before the field, read as the key's number.
|
||||
type tierProgress struct {
|
||||
Tier int64 `json:"tier"`
|
||||
Since time.Time `json:"since"`
|
||||
Movers []tierMover `json:"movers"`
|
||||
Source int64 `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
|
||||
func tierKey(pipeline string) string { return tierPrefix + pipeline }
|
||||
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form
|
||||
// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's
|
||||
// is (a module's end, `<module>.<branch>`, is none).
|
||||
func isTierKey(key string) bool {
|
||||
rest, ok := strings.CutPrefix(key, tierPrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := pipelinekey.Parse(rest)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
|
||||
// lowest of theirs, since now (H-A-277 (1)). A pipeline heard again adds to the progress that stands each mover it does not hold,
|
||||
// at the mover's tier, and marks versioned each held mover whose version is now written (H-A-270, H-A-273): its want is written
|
||||
// when the progress reaches its tier, or at once when that tier ran already, never out of tier order.
|
||||
func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover, versioned map[string]bool) error {
|
||||
for {
|
||||
t, rev, ok := e.tiersOf(pipeline)
|
||||
if !ok {
|
||||
if len(movers) == 0 {
|
||||
return nil
|
||||
}
|
||||
first := movers[0].Tier
|
||||
for _, m := range movers {
|
||||
first = min(first, m.Tier)
|
||||
}
|
||||
t = tierProgress{Tier: first, Since: e.clock().UTC(), Source: source}
|
||||
}
|
||||
moved := false
|
||||
for i := range t.Movers {
|
||||
if t.Movers[i].NoVersion && versioned[t.Movers[i].Module] {
|
||||
t.Movers[i].NoVersion, moved = false, true
|
||||
}
|
||||
}
|
||||
for _, m := range movers {
|
||||
if !slices.ContainsFunc(t.Movers, func(h tierMover) bool { return h.Module == m.Module }) {
|
||||
t.Movers, moved = append(t.Movers, m), true
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
return nil
|
||||
}
|
||||
value, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.st.swap(tierKey(pipeline), value, rev)
|
||||
if errors.Is(err, errSwapped) {
|
||||
if _, _, held := e.tiersOf(pipeline); !ok && !held {
|
||||
return nil // a key that stands but does not read is left as it is
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == nil && ok {
|
||||
e.logf("pipeline %s: a mover whose version came after its end is taken into the tier progress", pipeline)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (e *pipelineEnds) clock() time.Time {
|
||||
if e.now != nil {
|
||||
return e.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// tiersOf is the tier progress of pipeline, with its revision; false when none stands.
|
||||
func (e *pipelineEnds) tiersOf(pipeline string) (tierProgress, uint64, bool) {
|
||||
raw, rev, err := e.st.get(tierKey(pipeline))
|
||||
var t tierProgress
|
||||
if err != nil || raw == nil || json.Unmarshal(raw, &t) != nil {
|
||||
return tierProgress{}, 0, false
|
||||
}
|
||||
return t, rev, true
|
||||
}
|
||||
|
||||
// tierWritten says whether the want of an end may be written: its pipeline's tier progress reached the end's rank, or no
|
||||
// progress stands for its pipeline (every tier ran).
|
||||
func (e *pipelineEnds) tierWritten(end End) bool {
|
||||
t, _, ok := e.tiersOf(end.Pipeline)
|
||||
return !ok || end.Rank <= t.Tier
|
||||
}
|
||||
|
||||
// tierBound is the setting tier-bound; zero or less is its default.
|
||||
func (p *passer) tierBoundOf() time.Duration {
|
||||
if p.tierBound > 0 {
|
||||
return p.tierBound
|
||||
}
|
||||
return defaultTierBound
|
||||
}
|
||||
|
||||
// tiers moves each pipeline's tier progress (design 47 H §751): a tier whose placed movers all run on every present placing machine
|
||||
// gives way to the next, the last to the end said `running`; a tier past tier-bound is late. placedOn is, by module, the nodes
|
||||
// placing it on branch. It returns the earliest deadline still to come (zero: none), for one timer to wake the pass at.
|
||||
func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []string) time.Time {
|
||||
if p.ends == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
ks, err := p.ends.st.keys()
|
||||
if err != nil {
|
||||
p.logf("the tier progress cannot be listed: %v", err)
|
||||
return time.Time{}
|
||||
}
|
||||
sort.Strings(ks)
|
||||
now := p.clock().UTC()
|
||||
bound := p.tierBoundOf()
|
||||
var next time.Time
|
||||
for _, key := range ks {
|
||||
if !isTierKey(key) {
|
||||
continue
|
||||
}
|
||||
pipeline := strings.TrimPrefix(key, tierPrefix)
|
||||
pk, err := pipelinekey.Parse(pipeline)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10)
|
||||
for {
|
||||
t, rev, ok := p.ends.tiersOf(pipeline)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier)
|
||||
late := !now.Before(t.Since.Add(bound))
|
||||
if late {
|
||||
p.tierLate(pipeline, branch, number, t, notRunning)
|
||||
break
|
||||
}
|
||||
if len(notRunning) > 0 {
|
||||
if d := t.Since.Add(bound); next.IsZero() || d.Before(next) {
|
||||
next = d
|
||||
}
|
||||
break
|
||||
}
|
||||
// The tier runs: the next tier's wants are written, or every tier ran.
|
||||
later := int64(0)
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier > t.Tier && (later == 0 || m.Tier < later) {
|
||||
later = m.Tier
|
||||
}
|
||||
}
|
||||
if later == 0 {
|
||||
if err := p.ends.st.drop(key); err != nil {
|
||||
p.logf("the tier progress %s is not deleted: %v", key, err)
|
||||
break
|
||||
}
|
||||
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
break
|
||||
}
|
||||
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source})
|
||||
if err == nil {
|
||||
err = p.ends.st.swap(key, value, rev)
|
||||
}
|
||||
if errors.Is(err, errSwapped) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the tier progress %s is not moved to tier %d: %v", key, later, err)
|
||||
break
|
||||
}
|
||||
p.logf("pipeline %s: tier %d runs on every present placing machine, so tier %d's wants are written", pipeline, t.Tier, later)
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// notRunning is, of tier's movers in t whose kept end still names pipeline, those not running at their fingerprint on a present
|
||||
// placing machine, with those machines. A mover a later pipeline moved counts as run; one placed nowhere counts as run.
|
||||
func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) []string, pipeline, branch string, t tierProgress, tier int64) []meshwants.NotRunning {
|
||||
var out []meshwants.NotRunning
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier != tier {
|
||||
continue
|
||||
}
|
||||
if m.NoVersion {
|
||||
// A mover whose version is not yet written holds its tier, under the tier bound (H-A-277 (1)).
|
||||
machines := []string{}
|
||||
for _, node := range placedOn(m.Module, branch) {
|
||||
if online, known := n.Online[node]; !known || online {
|
||||
machines = append(machines, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
|
||||
continue
|
||||
}
|
||||
if end, ok := p.ends.endOf(m.Module, branch); !ok || end.Pipeline != pipeline {
|
||||
continue // a later pipeline moved it
|
||||
}
|
||||
var machines []string
|
||||
for _, node := range placedOn(m.Module, branch) {
|
||||
if online, known := n.Online[node]; known && !online {
|
||||
continue // a machine away is not waited for
|
||||
}
|
||||
r, ok := n.Running[node][m.Module]
|
||||
if !ok || r.Stopped || r.Fingerprint != m.Fp {
|
||||
machines = append(machines, node)
|
||||
}
|
||||
}
|
||||
if len(machines) > 0 {
|
||||
sort.Strings(machines)
|
||||
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late,
|
||||
// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is
|
||||
// written, until no later tier's mover is kept at this pipeline any more.
|
||||
func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) {
|
||||
o := tierLateFinding(pipeline, t.Tier, notRunning)
|
||||
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
|
||||
if len(notRunning) > 0 {
|
||||
if !p.said[key+".raised"] {
|
||||
if p.findings.Put != nil {
|
||||
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.said[key+".raised"] = true
|
||||
delete(p.said, key+".cleared")
|
||||
p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary)
|
||||
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
|
||||
}
|
||||
} else if !p.said[key+".cleared"] {
|
||||
if p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(key); err != nil {
|
||||
p.logf("the finding %s is not deleted: %v", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.said[key+".cleared"] = true
|
||||
delete(p.said, key+".raised")
|
||||
}
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier <= t.Tier {
|
||||
continue
|
||||
}
|
||||
if end, ok := p.ends.endOf(m.Module, branch); ok && end.Pipeline == pipeline {
|
||||
return // a later tier's mover still kept at this pipeline: its want stays unwritten
|
||||
}
|
||||
}
|
||||
if len(notRunning) == 0 {
|
||||
if err := p.ends.st.drop(tierKey(pipeline)); err != nil {
|
||||
p.logf("the tier progress of %s is not deleted: %v", pipeline, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tierLateFinding is the finding tier-late of pipeline's tier, naming the movers and machines not running (design 47 H §751).
|
||||
func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunning) assign.Observed {
|
||||
var named []string
|
||||
for _, nr := range notRunning {
|
||||
named = append(named, fmt.Sprintf("%s on %s", nr.Module, strings.Join(nr.Machines, ", ")))
|
||||
}
|
||||
summary := fmt.Sprintf("pipeline %s: tier %d does not run within tier-bound: %s", pipeline, tier, strings.Join(named, "; "))
|
||||
return assign.Observed{Scope: mcc.ScopeDelivery, ID: pipeline, Kind: kindTierLate, Severity: mcseat.FindingSeverityWarning,
|
||||
Summary: summary, Headline: "A delivery's tier does not run in time",
|
||||
Explanation: summary + ". The assigner writes no later tier of this pipeline; the delivery ends here.",
|
||||
Source: "assigner." + kindTierLate}
|
||||
}
|
||||
|
||||
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its
|
||||
// revision the number of the pipeline's key `<branch>.<revision>`, its source the one the pipeline-ended named (design 47
|
||||
// H-A-214, H-A-216), the key's number for a progress kept before it held one.
|
||||
func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) {
|
||||
var revision int64
|
||||
_, _ = fmt.Sscan(number, &revision)
|
||||
if source <= 0 {
|
||||
source = revision
|
||||
}
|
||||
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning}
|
||||
event := meshwants.EventPipelineWanted + "." + pipeline
|
||||
if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil {
|
||||
p.logf("pipeline-wanted of %s is not said: %v", pipeline, err)
|
||||
}
|
||||
}
|
||||
|
||||
// tierDeadline sets the one timer that wakes the pass at the earliest tier's deadline (H-A-56); zero stops it.
|
||||
func (p *passer) tierDeadline(at time.Time) {
|
||||
if p.tierWake == nil {
|
||||
return
|
||||
}
|
||||
if p.tierTimer != nil {
|
||||
p.tierTimer.Stop()
|
||||
}
|
||||
if at.IsZero() {
|
||||
return
|
||||
}
|
||||
p.tierTimer = time.AfterFunc(max(at.Sub(p.clock()), 0), p.tierWake)
|
||||
}
|
||||
|
||||
// tierBoundSetting is the setting tier-bound in minutes (design 47 H §751), read at the start from the file the manifest declares
|
||||
// with the content ${config:tier-bound}; zero, its default, for a file missing or empty.
|
||||
func tierBoundSetting(getenv func(string) string) (time.Duration, error) {
|
||||
minutes, err := settingFile(getenv("MESH_ASSIGNER_TIER_BOUND_FILE"))
|
||||
return time.Duration(minutes) * tierBoundUnit, err
|
||||
}
|
||||
|
||||
// tierBoundUnit is the unit of the setting tier-bound, a minute; the whole test sets a shorter one, so a tier passes its bound
|
||||
// without a test waiting minutes.
|
||||
var tierBoundUnit = time.Minute
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A7/AC1, A7/AC2 (design 47 H §751, H-A-56): a pipeline of a provider (tier 1) and its consumer (tier 2) writes the provider's
|
||||
// wants first and the consumer's only once the provider runs at its fingerprint on every present placing machine (one machine
|
||||
// away is not waited for), then says pipeline-wanted running and drops its progress; a provider that never runs is late at
|
||||
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
|
||||
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
|
||||
// H-A-485: a pipeline started again at the head, keyed `<branch>.<revision>.again.<n>`, moves tier by tier as a take-in's does.
|
||||
for _, c := range []struct {
|
||||
runs bool
|
||||
pipeline string
|
||||
}{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} {
|
||||
runs, pipeline := c.runs, c.pipeline
|
||||
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
|
||||
p.now = func() time.Time { return now }
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
findings := map[string]mcseat.Finding{}
|
||||
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { findings[views.FindingKey(f.Scope, f.ID, f.Kind)] = f; return nil },
|
||||
Delete: func(key string) error { delete(findings, key); return nil }}
|
||||
var said []meshwants.PipelineWanted
|
||||
p.publishWants = func(event string, body any, _ string) error {
|
||||
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
|
||||
if event != "pipeline-wanted."+pipeline {
|
||||
t.Errorf("pipeline-wanted said as %s", event)
|
||||
}
|
||||
said = append(said, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for i, m := range []struct{ name, fp string }{{"db", db}, {"web", web}} {
|
||||
digest := strings.Repeat(string(rune('e'+i)), 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.fp, Manifest: digest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[digest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
}
|
||||
placeOn(t, p.w, "g14", "db", "web")
|
||||
placeOn(t, p.w, "ace", "db", "web")
|
||||
p.w.heardPresence("ace", true, start) // away: not waited for
|
||||
p.w.mu.Lock()
|
||||
p.w.runningHeard = true
|
||||
p.w.mu.Unlock()
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
|
||||
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
|
||||
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
|
||||
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
|
||||
e.Source, e.Pipeline = 4, pipeline
|
||||
if _, err := p.ends.take(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wanted := func(node, module string) bool {
|
||||
key, _ := mwc.WantKey(node, module)
|
||||
_, ok := wantsKV.values[key]
|
||||
return ok
|
||||
}
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values)
|
||||
}
|
||||
if runs {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
|
||||
if err := p.w.hearRunning("running.g14.db", raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = start.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "web") || !wanted("ace", "web") {
|
||||
t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values)
|
||||
}
|
||||
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
|
||||
_ = p.w.hearRunning("running.g14.web", raw)
|
||||
p.wants(context.Background())
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
|
||||
t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings)
|
||||
}
|
||||
if _, _, ok := p.ends.tiersOf(pipeline); ok {
|
||||
t.Error("the tier progress stands after every tier ran")
|
||||
}
|
||||
continue
|
||||
}
|
||||
now = start.Add(29 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
if len(findings) != 0 || len(said) != 0 {
|
||||
t.Fatalf("late before tier-bound: %v, %+v", findings, said)
|
||||
}
|
||||
now = start.Add(30 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"]
|
||||
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
|
||||
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
|
||||
}
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" ||
|
||||
said[0].Revision != 5 || said[0].Source != 4 {
|
||||
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
|
||||
}
|
||||
// A tier progress kept before it held its source says the key's number as both (H-A-216).
|
||||
p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
|
||||
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
|
||||
}
|
||||
if wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Error("a consumer's want is written after its provider's tier was late")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-270, H-A-273, H-A-277 (1): a pipeline's tier progress holds every passed mover, versioned or not, and a tier with a mover
|
||||
// whose version is not yet written does not run until it is: no later tier is wanted out of order. When the lowest tier's version
|
||||
// comes last, nothing is wanted before it; with db (1) and api (3) versioned and web (2) late, api is wanted only once web runs.
|
||||
func TestALateMoverHoldsItsTier(t *testing.T) {
|
||||
type step struct {
|
||||
versions map[string]string // a pipeline-ended taken, each mover's version ("": none yet)
|
||||
runs string // or: this module runs on g14
|
||||
wanted string // then: the modules wanted, in order of the list
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
steps []step
|
||||
}{
|
||||
{"the lowest tier's version comes last", []step{
|
||||
{versions: map[string]string{"db": "", "web": "0.1.2", "api": "0.3.0"}, wanted: ""},
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db"},
|
||||
{runs: "db", wanted: "db,web"},
|
||||
{runs: "web", wanted: "db,web,api"},
|
||||
}},
|
||||
{"a middle tier's version comes late", []step{
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "", "api": "0.3.0"}, wanted: "db"},
|
||||
{runs: "db", wanted: "db"},
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db,web"},
|
||||
{runs: "web", wanted: "db,web,api"},
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fps := map[string]string{}
|
||||
tiers := map[string]int64{"db": 1, "web": 2, "api": 3}
|
||||
for i, m := range []string{"db", "web", "api"} {
|
||||
fps[m] = meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i)), 64)
|
||||
}
|
||||
start := time.Date(2026, 11, 21, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
|
||||
p.now = func() time.Time { return now }
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(string, any, string) error { return nil }
|
||||
for i, m := range []string{"db", "web", "api"} {
|
||||
digest := strings.Repeat(string(rune('1'+i)), 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fps[m], Manifest: digest, Repository: "novox/" + m, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m, mod)
|
||||
p.w.manifests[digest] = parsed{module: m, manifest: catalogue.Manifest{Module: m}}
|
||||
}
|
||||
placeOn(t, p.w, "g14", "db", "web", "api")
|
||||
p.w.mu.Lock()
|
||||
p.w.runningHeard = true
|
||||
p.w.mu.Unlock()
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
|
||||
for i, s := range c.steps {
|
||||
if s.versions != nil {
|
||||
var movers []mpr.MoverEnded
|
||||
for _, m := range []string{"db", "web", "api"} {
|
||||
movers = append(movers, mover(m, fps[m], s.versions[m], tiers[m], mpr.StepResultOutcomePassed))
|
||||
}
|
||||
if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, movers...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: s.runs, Fingerprint: fps[s.runs]})
|
||||
if err := p.w.hearRunning("running.g14."+s.runs, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
var got []string
|
||||
for _, m := range []string{"db", "web", "api"} {
|
||||
key, _ := mwc.WantKey("g14", m)
|
||||
if _, ok := wantsKV.values[key]; ok {
|
||||
got = append(got, m)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, ",") != s.wanted {
|
||||
t.Fatalf("step %d: wanted %v, want %q", i+1, got, s.wanted)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
|
||||
"encoding/json"
|
||||
@@ -22,13 +21,11 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
@@ -36,9 +33,6 @@ import (
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
@@ -215,12 +209,6 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
if arg(args, "how") != "" {
|
||||
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
|
||||
}
|
||||
if step := arg(args, "step"); step != "" {
|
||||
if step != meshwants.WaitStepDelivery {
|
||||
return nil, fmt.Errorf("step is %s, the one wait released by its step, not %q", meshwants.WaitStepDelivery, step)
|
||||
}
|
||||
return s.doReleaseDelivery(args, by) // a dependent's delivery wait (delivered.go, design 47 G §376, A-246)
|
||||
}
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := puts.Key(node, module)
|
||||
if err != nil {
|
||||
@@ -353,12 +341,10 @@ func (s *putTools) tools() []stdio.Tool {
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
|
||||
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
|
||||
"failed (the failing check is the delivery's), rejudge (judge it again now), or now (the entry waiting for an acting part's new version moves with the running one); written in the hand-act log first",
|
||||
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
|
||||
"failed (the failing check is the delivery's), or rejudge (judge it again now); written in the hand-act log first"},
|
||||
[]string{"module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
s.waitsTool(),
|
||||
s.deliveredAgainTool(),
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
@@ -403,7 +389,7 @@ func serveTools() error {
|
||||
}
|
||||
}()
|
||||
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
|
||||
gate := seatlist.NewPart("assigner", seatListAgainCallers)
|
||||
gate := seatlist.NewPart("assigner")
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
|
||||
@@ -417,17 +403,23 @@ func serveTools() error {
|
||||
return stdio.SeatPublish(wantsEventSubject(e), b, id)
|
||||
})
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
|
||||
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
|
||||
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
|
||||
seatVerbs.watched = wants.watched
|
||||
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
|
||||
tools = append(tools, wants.tools()...)
|
||||
tools = append(tools, inputsRead(&inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}))
|
||||
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
|
||||
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
|
||||
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
|
||||
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
|
||||
// What each node's engine reads, as the assigner uses it: its own state engine-reads (design 47 H-A-104).
|
||||
tools = append(tools, engineReadsTools(runtimeStore{})...)
|
||||
tools = append(tools, writtenTools(runtimeStore{})...)
|
||||
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
|
||||
tools = append(tools, whyTools(runtimeStore{})...)
|
||||
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
|
||||
tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...)
|
||||
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777)
|
||||
tools = append(tools, keepNowTools(stdio.Ask)...)
|
||||
tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2))
|
||||
return stdio.Serve("", tools)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat
|
||||
// mesh-wants, asked by the runner for each mover before its composition step. One run per take-in revision, a key
|
||||
// `<pipeline>.validate` of the state composition-runs in the CompositionRun form with no machine, which every mover's step
|
||||
// reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from
|
||||
// the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the
|
||||
// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input.
|
||||
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case
|
||||
// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed.
|
||||
|
||||
const (
|
||||
// validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's
|
||||
// cmd/mesh-judge, design 47 H-A-34).
|
||||
validatorPackage = "mesh-delivery-indexer/cmd/mesh-judge"
|
||||
// validateKeySuffix ends the key of a revision's validate run, `<pipeline>.validate` (H-A-108).
|
||||
validateKeySuffix = "." + meshwants.StepRunStepValidate
|
||||
// beforeCase is the validator's case of the revision's base, beside changeCase (H-A-108 (3), H-A-151, H-A-154).
|
||||
beforeCase = "before"
|
||||
// The reasons a validate step fails with, the first word of its said (§745).
|
||||
reasonManifestRefused = "manifest-refused"
|
||||
reasonCycle = "cycle"
|
||||
// betweenManifests is the validator's name for what it judges between manifests (the catalogue as a whole).
|
||||
betweenManifests = "(between manifests)"
|
||||
)
|
||||
|
||||
// validateKeyOf is the key of the validate run of the step's take-in revision.
|
||||
func validateKeyOf(step meshwants.StepRun) (string, error) {
|
||||
if step.Pipeline == "" {
|
||||
return "", errors.New("the step names no pipeline, so no validate run is named")
|
||||
}
|
||||
return step.Pipeline + validateKeySuffix, nil
|
||||
}
|
||||
|
||||
// isValidateRun says whether a run of the state is a revision's validate run: one with no machine.
|
||||
func isValidateRun(run meshwants.CompositionRun) bool { return run.Machine == "" }
|
||||
|
||||
// validate runs one mover's validate step: it joins or starts the revision's validate run and judges the step from its end.
|
||||
func (r *compositionRun) validate(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
j, err := r.runs.joinAt(key, readerOf(id, step), candidate, "", buildstep.NewID())
|
||||
if err != nil {
|
||||
return couldNotRun("the validate run " + key + " is not joined: " + err.Error())
|
||||
}
|
||||
if j.started {
|
||||
go r.startValidate(ctx, key, j.run, step)
|
||||
}
|
||||
return r.readValidate(ctx, key, step)
|
||||
}
|
||||
|
||||
// resumeValidate judges a validate step taken up at the assigner's start from its revision's run.
|
||||
func (r *compositionRun) resumeValidate(ctx context.Context, _ string, step meshwants.StepRun) stepOutcome {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
return r.readValidate(ctx, key, step)
|
||||
}
|
||||
|
||||
// readValidate waits for the end of the validate run of key and judges the step from it.
|
||||
func (r *compositionRun) readValidate(ctx context.Context, key string, step meshwants.StepRun) stepOutcome {
|
||||
setShared(ctx, r.runs.sharedOf([]string{key}, nil))
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
run := r.runs.await(ctx, []string{key})[key]
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
|
||||
out := r.judgeValidate(ctx, step, run)
|
||||
out.Shared = r.runs.sharedOf([]string{key}, map[string]meshwants.CompositionRun{key: run})
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeValidate is a step's end from its revision's validate run: could not run with the run's words; failed `cycle` naming
|
||||
// the cycle; failed `manifest-refused` naming every problem the revision causes; else passed.
|
||||
func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepRun, run meshwants.CompositionRun) stepOutcome {
|
||||
if run.Ended == nil {
|
||||
return couldNotRun("the step stopped before its validate run ended")
|
||||
}
|
||||
if run.Outcome != meshwants.CompositionRunOutcomePassed {
|
||||
return couldNotRun(run.Said)
|
||||
}
|
||||
var v validateVerdict
|
||||
if raw, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(raw, &v) != nil {
|
||||
return couldNotRun("the validate run's verdict does not read")
|
||||
}
|
||||
if len(v.Cycle) > 0 {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: reasonCycle + ": " + cycleWords(v.Cycle)}
|
||||
}
|
||||
half, err := r.revision(ctx, step)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's catalog revision is not read: " + err.Error())
|
||||
}
|
||||
// A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151).
|
||||
empty := ""
|
||||
if half.Revision.Base <= 0 {
|
||||
empty = emptyBeforeWords
|
||||
}
|
||||
if problems := causedProblems(v.Change, v.Before); len(problems) > 0 {
|
||||
said := reasonManifestRefused + ": " + strings.Join(problems, "; ")
|
||||
if empty != "" {
|
||||
said += " (" + empty + ")"
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: said}
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed, Said: empty}
|
||||
}
|
||||
|
||||
// emptyBeforeWords say, in a validate step's end, that its revision names no base: the branch's first take-in, judged against
|
||||
// an empty case before (H-A-151, H-A-154).
|
||||
const emptyBeforeWords = "the case before is empty: the revision names no base, the branch's first take-in"
|
||||
|
||||
// validateVerdict is a validate run's verdict as the run keeps it: the cycle the revision makes, or the validator's two cases.
|
||||
type validateVerdict struct {
|
||||
Cycle []meshcatalog.ArtifactRef `json:"cycle,omitempty"`
|
||||
Change composer.CaseVerdict `json:"change"`
|
||||
Before composer.CaseVerdict `json:"before"`
|
||||
}
|
||||
|
||||
// asVerdict is v as a run's verdict.
|
||||
func (v validateVerdict) asVerdict() (map[string]any, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
return out, json.Unmarshal(raw, &out)
|
||||
}
|
||||
|
||||
// cycleWords names a cycle as the artifacts it passes through.
|
||||
func cycleWords(cycle []meshcatalog.ArtifactRef) string {
|
||||
words := make([]string, len(cycle))
|
||||
for i, a := range cycle {
|
||||
words[i] = string(a)
|
||||
}
|
||||
return strings.Join(words, " → ")
|
||||
}
|
||||
|
||||
// before is the case before of the step's revision half: the revision its base names, the newest kept revision below it that
|
||||
// its take-in pinned, read through the seat mesh-catalog's verb revision (H-A-108 (3), H-A-151, H-A-154); empty where the
|
||||
// revision names no base, the branch's first take-in.
|
||||
func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, half mcc.Half) (mcc.Half, error) {
|
||||
if half.Revision.Base <= 0 {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}, nil
|
||||
}
|
||||
base := step
|
||||
base.Revision = half.Revision.Base
|
||||
return r.revision(ctx, base)
|
||||
}
|
||||
|
||||
// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between
|
||||
// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the
|
||||
// revision alters that manifest or not (assigner_validate_before answers those).
|
||||
func causedProblems(change, before composer.CaseVerdict) []string {
|
||||
return problemsBy(change, before, false)
|
||||
}
|
||||
|
||||
// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814).
|
||||
func uncountedProblems(change, before composer.CaseVerdict) []string {
|
||||
return problemsBy(change, before, true)
|
||||
}
|
||||
|
||||
// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had).
|
||||
func problemsBy(change, before composer.CaseVerdict, had bool) []string {
|
||||
var out []string
|
||||
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
|
||||
read := change.Manifests[module]
|
||||
if read.Accepted || read.Refused == "" {
|
||||
continue
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, p := range strings.Split(before.Manifests[module].Refused, "\n") {
|
||||
held[p] = true
|
||||
}
|
||||
var named []string
|
||||
for _, p := range strings.Split(read.Refused, "\n") {
|
||||
if p != "" && held[p] == had {
|
||||
named = append(named, p)
|
||||
}
|
||||
}
|
||||
if len(named) == 0 {
|
||||
continue
|
||||
}
|
||||
what := module
|
||||
if module == betweenManifests {
|
||||
what = "between manifests"
|
||||
}
|
||||
out = append(out, what+": "+strings.Join(named, "; "))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// catalogEntry is the entry of the module that holds the seat mesh-catalog, or false.
|
||||
func catalogEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
|
||||
for _, name := range slices.Sorted(maps.Keys(entries)) {
|
||||
for _, s := range entries[name].Serves {
|
||||
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), meshcatalog.Seat+".") {
|
||||
return entries[name], true
|
||||
}
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, false
|
||||
}
|
||||
|
||||
// manifestsOf are the half's manifests whole, as the validator's case holds them.
|
||||
func manifestsOf(half mcc.Half) (map[string]composer.Manifest, error) {
|
||||
out := map[string]composer.Manifest{}
|
||||
for name, e := range half.Entries {
|
||||
if len(e.Manifest) == 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
|
||||
}
|
||||
out[name] = composer.Manifest{Raw: raw}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// startValidate runs the validate run the step started: the revision's cycle ends it at once; else the validator's request
|
||||
// is kept once in `inputs`, its program and input recorded on the run, and one run ask put on the Go build seat. A fault ends
|
||||
// the run could not run.
|
||||
func (r *compositionRun) startValidate(ctx context.Context, key string, run meshwants.CompositionRun, step meshwants.StepRun) {
|
||||
ask := run.Ask
|
||||
end := func(outcome, said string, verdict map[string]any) {
|
||||
if _, err := r.runs.endRun(key, ask, outcome, said, verdict); err != nil {
|
||||
r.logf("the validate run %s is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
couldNot := func(said string) { end(meshwants.CompositionRunOutcomeCouldNotRun, said, nil) }
|
||||
half, err := r.revision(ctx, step)
|
||||
if err != nil {
|
||||
couldNot("the step's catalog revision is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if cycle := graph.RefusedCycle(half.Entries); cycle != nil {
|
||||
verdict, err := validateVerdict{Cycle: cycle}.asVerdict()
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
end(meshwants.CompositionRunOutcomePassed, "", verdict)
|
||||
return
|
||||
}
|
||||
before, err := r.before(ctx, step, half)
|
||||
if err != nil {
|
||||
couldNot("the base of the case before is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
entry, ok := catalogEntry(half.Entries)
|
||||
if !ok {
|
||||
couldNot(fmt.Sprintf("the catalog revision %d holds no module claiming the seat %s, so no manifest validator is built", half.Revision.Revision, meshcatalog.Seat))
|
||||
return
|
||||
}
|
||||
program, err := programOfEntry(entry)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
program.Package = validatorPackage
|
||||
seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
change, err := manifestsOf(half)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
prior, err := manifestsOf(before)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
// No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)).
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}},
|
||||
Seats: seats, SeatList: list}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
couldNot("its request cannot be encoded: " + err.Error())
|
||||
return
|
||||
}
|
||||
digest, err := r.keeper.keep(ask, raw)
|
||||
if err != nil {
|
||||
couldNot("its request is not kept: " + err.Error())
|
||||
return
|
||||
}
|
||||
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: entry.Fp}
|
||||
c.Input = digest
|
||||
return true
|
||||
}); err != nil {
|
||||
couldNot("the run's program and input are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the manifest validator of " + step.Pipeline + " at " + program.Commit},
|
||||
Packages: pinsOfEntry(entry)} // the validator is built against its entry's pins at the step's revision (H-A-203, H-A-192, H-A-205)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
couldNot("its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// validateRunVerdict reads the validator's answer: its two cases as the run keeps them, or why it is none.
|
||||
func validateRunVerdict(answer []byte) (map[string]any, string) {
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(answer, &v); err != nil {
|
||||
return nil, "the manifest validator answered no verdict: " + err.Error()
|
||||
}
|
||||
if v.V != composer.Version {
|
||||
return nil, fmt.Sprintf("the manifest validator answered a verdict of version %d, the assigner reads %d", v.V, composer.Version)
|
||||
}
|
||||
var out validateVerdict
|
||||
found := 0
|
||||
for _, c := range v.Cases {
|
||||
switch c.Name {
|
||||
case changeCase:
|
||||
out.Change, found = c, found+1
|
||||
case beforeCase:
|
||||
out.Before, found = c, found+1
|
||||
}
|
||||
}
|
||||
if found != 2 {
|
||||
return nil, "the manifest validator's verdict does not hold both the cases " + changeCase + " and " + beforeCase
|
||||
}
|
||||
verdict, err := out.asVerdict()
|
||||
if err != nil {
|
||||
return nil, "the manifest validator's verdict cannot be kept: " + err.Error()
|
||||
}
|
||||
return verdict, ""
|
||||
}
|
||||
|
||||
// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814).
|
||||
const validateBeforeTool = "assigner_validate_before"
|
||||
|
||||
// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too,
|
||||
// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps.
|
||||
func validateBefore(st store, pipeline, module string) (map[string]any, error) {
|
||||
key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, ok, err := st.get(buckets.CompositionRunsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no validate run %s is kept", key)
|
||||
}
|
||||
var run meshwants.CompositionRun
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
return nil, fmt.Errorf("the validate run %s does not read: %w", key, err)
|
||||
}
|
||||
var v validateVerdict
|
||||
if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil {
|
||||
return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key)
|
||||
}
|
||||
change, before := v.Change, v.Before
|
||||
if module = strings.TrimSpace(module); module != "" {
|
||||
change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}}
|
||||
}
|
||||
uncounted := uncountedProblems(change, before)
|
||||
if uncounted == nil {
|
||||
uncounted = []string{}
|
||||
}
|
||||
return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil
|
||||
}
|
||||
|
||||
// validateBeforeTools are the module's tool of the validate step's before case.
|
||||
func validateBeforeTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " +
|
||||
"validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " +
|
||||
"for one module when named. Read-only.",
|
||||
Input: schema(map[string]string{"pipeline": "the pipeline's key, <branch>.<revision>", "module": "one module; absent: every one"}, []string{"pipeline"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return validateBefore(st, arg(args, "pipeline"), arg(args, "module"))
|
||||
}}}
|
||||
}
|
||||
|
||||
// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443).
|
||||
const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList
|
||||
|
||||
// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by
|
||||
// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them,
|
||||
// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb
|
||||
// seat-list hands, asked as a kept call under the run's ask.
|
||||
func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) {
|
||||
if len(defs) == 0 {
|
||||
return r.seatList(ask)
|
||||
}
|
||||
sorted := slices.Clone(defs)
|
||||
slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) })
|
||||
seats, err := r.seats(ctx, sorted)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
raw, err := json.Marshal(seats)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err)
|
||||
}
|
||||
var list []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &list); err != nil {
|
||||
return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err)
|
||||
}
|
||||
return list, seatListDigest(raw), nil
|
||||
}
|
||||
|
||||
// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex.
|
||||
func seatListDigest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in
|
||||
// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest
|
||||
// judges the same bytes.
|
||||
func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) {
|
||||
what := "the trunk's seat list is not read through " + seatListKey
|
||||
if r.callFile == nil {
|
||||
return nil, "", errors.New(what + ": the assigner holds no chunked call")
|
||||
}
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
return nil, "", fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey,
|
||||
meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])})
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
var a struct {
|
||||
meshcatalog.SeatListAnswer
|
||||
File string `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err)
|
||||
}
|
||||
if a.Digest == "" {
|
||||
return nil, "", errors.New(what + ": its answer names no digest")
|
||||
}
|
||||
file, err := base64.StdEncoding.DecodeString(a.File)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err)
|
||||
}
|
||||
if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest {
|
||||
return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file),
|
||||
seatListDigest(file), a.Size, a.Digest)
|
||||
}
|
||||
var list []json.RawMessage
|
||||
if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 {
|
||||
return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest)
|
||||
}
|
||||
return list, a.Digest, nil
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate
|
||||
// run, `<pipeline>.validate` of the state composition-runs; a manifest the revision alters that names an unknown field fails
|
||||
// it `manifest-refused` naming the problem, a problem of a manifest it leaves as it was does not, and the validator is the
|
||||
// revision's own entry of the module holding the seat mesh-catalog, on the Go build seat; a revision whose entries make a cycle
|
||||
// fails it `cycle` naming the cycle, with no validator asked. The case before is the revision its base names (H-A-151, H-A-154:
|
||||
// the newest kept below it, which the stand-in names as the indexer does); a revision naming no base, the branch's first take-in,
|
||||
// is judged against an empty case before, said in its end, and no revision below it is asked.
|
||||
func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix",
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient": "v0.1.4"}}
|
||||
entry := func(module, digest string, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
revisions := map[int64][]meshcatalog.CatalogEntry{
|
||||
6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")},
|
||||
7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")},
|
||||
3: {indexer},
|
||||
12: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
}
|
||||
// base is the newest kept revision below n, as the take-in names it; none below the first. Revision 12 names 7 with 8 to 11
|
||||
// never kept, so a case before read as the number minus one could not run (H-A-154).
|
||||
base := func(n int64) (b int64) {
|
||||
if n == 12 {
|
||||
return 7
|
||||
}
|
||||
for k := range revisions {
|
||||
if k < n && k > b {
|
||||
b = k
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
switch key {
|
||||
case mcc.RevisionVerb:
|
||||
var in meshcatalog.RevisionInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
entries, ok := revisions[in.Revision]
|
||||
if !ok || in.Branch != "main" {
|
||||
return nil, errors.New("asked another revision")
|
||||
}
|
||||
// Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)).
|
||||
var defs []meshcatalog.CatalogRevisionDefinitionsItem
|
||||
if in.Revision != 3 {
|
||||
defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision),
|
||||
Definitions: defs}, Entries: entries})
|
||||
case seatDefinitionsKey:
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}})
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
// The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file.
|
||||
trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`)
|
||||
listAnswer := func(file []byte, digest string, size int) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)})
|
||||
return raw
|
||||
}
|
||||
var listAsks []process.KeptAsk
|
||||
listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
|
||||
callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
in, _ := body.(meshcatalog.SeatListInput)
|
||||
if key != seatListKey || len(in.Call) != 32 {
|
||||
return nil, errors.New("asked " + key + " with the call " + in.Call)
|
||||
}
|
||||
listAsks = append(listAsks, ask)
|
||||
return listed, nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var asks []buildstep.StepAsk
|
||||
var requests []composer.Request
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
|
||||
r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf,
|
||||
put: func(_ context.Context, a buildstep.StepAsk) error {
|
||||
body, err := keeper.body(a.Run.Input.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := composer.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mu.Lock()
|
||||
asks, requests = append(asks, a), append(requests, req)
|
||||
mu.Unlock()
|
||||
answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{
|
||||
{Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}},
|
||||
{Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}})
|
||||
go func() {
|
||||
if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"},
|
||||
func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}}
|
||||
steps := &compositionSteps{validate: r.validate, run: r.run}
|
||||
if steps.howOf(meshwants.StepRunStepValidate, false) == nil || steps.howOf("check", false) != nil {
|
||||
t.Error("the validate step is not taken, or another step is")
|
||||
}
|
||||
step := func(module string, revision int64) meshwants.StepRun {
|
||||
return meshwants.StepRun{Step: meshwants.StepRunStepValidate, Pipeline: "main." + string(rune('0'+revision)), Branch: "main", Module: module,
|
||||
Revision: revision, Source: revision}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
outs := make([]stepOutcome, 2)
|
||||
var wg sync.WaitGroup
|
||||
for i, module := range []string{"web", "db"} {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); outs[i] = r.validate(ctx, "ask-"+module, step(module, 7)) }()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) ||
|
||||
strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems
|
||||
t.Errorf("mover %d's validate step ended %+v", i, o)
|
||||
}
|
||||
}
|
||||
// assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count.
|
||||
runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}}
|
||||
kept, _, _, _ := runs.read("main.7" + validateKeySuffix)
|
||||
runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept)
|
||||
before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"})
|
||||
if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) {
|
||||
t.Errorf("assigner_validate_before answered %+v, %v", before, err)
|
||||
}
|
||||
if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil {
|
||||
t.Error("assigner_validate_before answers a run not kept")
|
||||
}
|
||||
if len(asks) != 1 {
|
||||
t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks))
|
||||
}
|
||||
if p := asks[0].Run.Program; p.Commit != commitE3 || p.Dir != "modules/mesh-delivery-indexer" || p.Package != validatorPackage || len(asks[0].Run.Heads) != 0 {
|
||||
t.Errorf("the validator is built from %+v", asks[0].Run)
|
||||
}
|
||||
// The validator is built against the indexer entry's pins at the step's revision (H-A-203, H-A-192, H-A-205).
|
||||
if got := asks[0].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"] != "v0.1.4" {
|
||||
t.Errorf("the validator is built against the pins %v; want its entry's", got)
|
||||
}
|
||||
if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil {
|
||||
t.Errorf("the validator's request is %+v", requests[0])
|
||||
}
|
||||
// A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat
|
||||
// list is asked.
|
||||
if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 {
|
||||
t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own))
|
||||
}
|
||||
if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok {
|
||||
t.Error("the validate run is not the key <pipeline>.validate of composition-runs")
|
||||
}
|
||||
o := r.validate(ctx, "ask-a", step("a", 9))
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || o.Said != reasonCycle+": a:x → b:y → a:x" || len(asks) != 1 {
|
||||
t.Errorf("a revision making a cycle ended %+v (%d validator asks)", o, len(asks))
|
||||
}
|
||||
// The stand-in validator answers its one verdict: judged against the empty case before, web's problem is the revision's.
|
||||
if o := r.validate(ctx, "ask-first", step("indexer", 3)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 {
|
||||
t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks))
|
||||
}
|
||||
// Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest,
|
||||
// asked once as a kept call under the run's ask (H-A-441 (b), H-A-443).
|
||||
if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) ||
|
||||
len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 {
|
||||
t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList))
|
||||
}
|
||||
// Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
answer json.RawMessage
|
||||
}{
|
||||
{"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))},
|
||||
{"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)},
|
||||
{"no digest", listAnswer(trunkList, "", len(trunkList))},
|
||||
{"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))},
|
||||
{"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)},
|
||||
} {
|
||||
listed = row.answer
|
||||
if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) {
|
||||
t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err)
|
||||
}
|
||||
}
|
||||
listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
|
||||
// Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one
|
||||
// is not, and the case before holds base 7's manifests.
|
||||
if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour"` || len(asks) != 3 {
|
||||
t.Errorf("a revision naming base 7 below its number ended %+v (%d validator asks), want it judged against base 7", o, len(asks))
|
||||
} else if web := requests[2].Cases[1].Manifests["web"]; len(requests[2].Cases[1].Manifests) != 3 || !strings.Contains(string(web.Raw), "d-web-2") {
|
||||
t.Errorf("the case before of revision 12 is %+v, want base 7's manifests", requests[2].Cases[1].Manifests)
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -21,26 +24,6 @@ import (
|
||||
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
|
||||
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
|
||||
|
||||
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
|
||||
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
|
||||
// no node of module.
|
||||
func keptInvokes(g graph.Graph, module string) []string {
|
||||
n, ok := g[module]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, v := range n.Invokes {
|
||||
if slices.Contains(n.LeftOut, v) {
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
|
||||
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
|
||||
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
|
||||
@@ -50,14 +33,33 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
if len(g) == 0 {
|
||||
return nil
|
||||
}
|
||||
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if len(unread) > 0 {
|
||||
if p.unread.changed(unread) && len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// saidSet is the last set of names said, so a pass says a set again only when it changed.
|
||||
type saidSet struct {
|
||||
mu sync.Mutex
|
||||
last string
|
||||
}
|
||||
|
||||
// changed keeps names as the set said last and says whether it differs from the one before.
|
||||
func (s *saidSet) changed(names []string) bool {
|
||||
sorted := slices.Sorted(slices.Values(names))
|
||||
key := strings.Join(sorted, " ")
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if key == s.last {
|
||||
return false
|
||||
}
|
||||
s.last = key
|
||||
return true
|
||||
}
|
||||
|
||||
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
|
||||
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
|
||||
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
|
||||
@@ -108,3 +110,40 @@ func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
|
||||
}
|
||||
return got.manifest, true
|
||||
}
|
||||
|
||||
// servingHaves are the haves rule 8 counts (design 47 G A-426, A-426a): a node that runs a healthy build of a part serves what that
|
||||
// build serves, whatever its version and whether or not the catalog lists it, so a have held (its build has no catalog entry, a
|
||||
// put holds it) or waiting for a person's yes, whose build runs healthy, counts as running; a running statement that names a build
|
||||
// stopped, or not healthy, does not. counted is told each build so counted that its phase alone would have left out.
|
||||
func servingHaves(n assign.Nodes, counted func(node, module string, h nodeengine.HaveByNodeModuleValue)) map[string]map[string]nodeengine.HaveByNodeModuleValue {
|
||||
out := map[string]map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for node, haves := range n.Haves {
|
||||
out[node] = map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for module, h := range haves {
|
||||
if (h.Phase == nodeengineclient.PhaseHeld || h.Phase == nodeengineclient.PhaseWaitingYes) && h.Health == nodeengineclient.HealthOK {
|
||||
if r, ok := n.Running[node][module]; !ok || (!r.Stopped && (r.Healthy == nil || *r.Healthy)) {
|
||||
if counted != nil {
|
||||
counted(node, module, h)
|
||||
}
|
||||
h.Phase = nodeengineclient.PhaseRunning
|
||||
}
|
||||
}
|
||||
out[node][module] = h
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countedServing says once per node, module and build that rule 8 counted a build whose have is not in phase running as serving
|
||||
// its verbs (A-426a): the rule says which build it counted.
|
||||
func (p *passer) countedServing(node, module string, h nodeengine.HaveByNodeModuleValue) {
|
||||
key := "serving " + node + "." + module + "@" + h.Fp + "." + h.Phase
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
p.logf("%s on %s runs %s healthy in phase %s: counted as serving what that build serves, whatever its version (design 47 G A-426a)", module, node, shortHex(h.Fp), h.Phase)
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
|
||||
marks := state(views.MarksState)
|
||||
var out []stdio.Tool
|
||||
for _, t := range firstViewTools {
|
||||
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now))
|
||||
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
|
||||
}
|
||||
for _, t := range readTools {
|
||||
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// wantDropTool drops one want of a node for a module by hand (design 47 H §765, H-A-58): a held or failed want of the earlier
|
||||
// model goes at the clean state, its key deleted and the act written in the hand-act log; the next pass writes the want of the
|
||||
// new form.
|
||||
const wantDropTool = "assigner_want_drop"
|
||||
|
||||
// wantsState is the state the wants are kept in.
|
||||
const wantsState = "wants"
|
||||
|
||||
// dropWant deletes the want of module on node, the act first recorded through the seat mesh-hand-acts.
|
||||
func dropWant(st seatStore, ask func(string, any) (json.RawMessage, error), node, module, why, by string) (any, error) {
|
||||
switch {
|
||||
case node == "" || module == "":
|
||||
return nil, errors.New("name the node and the module whose want is dropped")
|
||||
case why == "":
|
||||
return nil, errors.New("a want dropped by hand says why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
|
||||
}
|
||||
key, err := mwc.WantKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, found, err := st.get(wantsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the assigner holds no want under %s", key)
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the drop of %s is not recorded, so the want stays: %w", key, err)
|
||||
}
|
||||
if err := st.del(wantsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the want %s is not dropped: %w", key, err)
|
||||
}
|
||||
return map[string]any{"key": key, "dropped": json.RawMessage(raw), "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// The kinds the all form counts what it dropped by (design 47 H-A-123).
|
||||
const (
|
||||
dropHeld = "held"
|
||||
dropCurrent = "current"
|
||||
dropUndecodable = "undecodable"
|
||||
)
|
||||
|
||||
// dropAllWants deletes every want key of the state wants, held or not, and every value of an earlier form the decoder refuses
|
||||
// (the all form, design 47 H-A-121 (2), H-A-123): each act recorded through the seat mesh-hand-acts first, as the one-item form
|
||||
// does; it answers what it dropped, counted per kind. A drop not recorded stops it, saying what went before.
|
||||
func dropAllWants(st seatStore, ask func(string, any) (json.RawMessage, error), why, by string) (any, error) {
|
||||
switch {
|
||||
case why == "":
|
||||
return nil, errors.New("a want dropped by hand says why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
|
||||
}
|
||||
all, err := st.keys(wantsState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
counted := map[string]int{dropHeld: 0, dropCurrent: 0, dropUndecodable: 0}
|
||||
dropped := map[string][]string{}
|
||||
answer := func() map[string]any {
|
||||
return map[string]any{"dropped": dropped, "counted": counted, "by": by}
|
||||
}
|
||||
for _, key := range all {
|
||||
if !strings.HasPrefix(key, "want.") {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := st.get(wantsState, key)
|
||||
if err != nil {
|
||||
return answer(), fmt.Errorf("the want %s cannot be read, so the drop stops there: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
kind := wantKind(key, raw)
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
|
||||
if err != nil {
|
||||
return answer(), fmt.Errorf("the drop of %s is not recorded, so it and the wants after it stay: %w", key, err)
|
||||
}
|
||||
if err := st.del(wantsState, key); err != nil {
|
||||
return answer(), fmt.Errorf("the want %s is not dropped: %w", key, err)
|
||||
}
|
||||
counted[kind]++
|
||||
dropped[kind] = append(dropped[kind], key+" (hand act "+taken.Log+")")
|
||||
}
|
||||
return answer(), nil
|
||||
}
|
||||
|
||||
// wantKind says how the all form counts a want: undecodable when its key or value is of no form the decoder takes, held when it
|
||||
// holds, current otherwise (design 47 H-A-123).
|
||||
func wantKind(key string, raw []byte) string {
|
||||
if _, err := mwc.ParseWantNodeKey(key); err == nil {
|
||||
if _, err := mwc.DecodeNodeWant(raw); err != nil {
|
||||
return dropUndecodable
|
||||
}
|
||||
return dropCurrent
|
||||
}
|
||||
if _, _, err := mwc.ParseWantKey(key); err != nil {
|
||||
return dropUndecodable
|
||||
}
|
||||
w, err := mwc.DecodeWant(raw)
|
||||
switch {
|
||||
case err != nil:
|
||||
return dropUndecodable
|
||||
case w.Hold:
|
||||
return dropHeld
|
||||
}
|
||||
return dropCurrent
|
||||
}
|
||||
|
||||
// wantDropTools is the tool over the runtime's states.
|
||||
func wantDropTools(st seatStore, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: wantDropTool, Description: "Drop one want by hand (design 47 H §765): the want of module on node is " +
|
||||
"deleted from the state wants, as the clean state drops a held or failed want of the earlier model; the next pass writes the " +
|
||||
"want of the new form. Recorded as a hand act first; refused with no want under the key. With all true and no node or module, " +
|
||||
"every want key goes, held or not, and every value of an earlier form the decoder refuses, each recorded as a hand act; it " +
|
||||
"answers what it dropped, counted per kind (held, current, undecodable).",
|
||||
Input: schema(map[string]string{"node": "the node's name", "module": "the module's name", "why": "why the want is dropped",
|
||||
"all": "true drops every want of the state wants, naming no node or module"}, []string{"why"}, "all"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if arg(a, "all") == "true" {
|
||||
if arg(a, "node") != "" || arg(a, "module") != "" {
|
||||
return nil, errors.New("the all form drops every want: name no node or module with it")
|
||||
}
|
||||
return dropAllWants(st, ask, arg(a, "why"), stdio.CallerOf(a))
|
||||
}
|
||||
return dropWant(st, ask, arg(a, "node"), arg(a, "module"), arg(a, "why"), stdio.CallerOf(a))
|
||||
}}}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
)
|
||||
|
||||
// A12/AC1 (design 47 H §765, H-A-58): assigner_want_drop with node, module and why records the act in the hand-act log first
|
||||
// and then deletes that want; with no want under the key, no why or no proven caller, and when the act is not recorded, nothing is
|
||||
// dropped.
|
||||
func TestAWantIsDroppedByHandAndRecorded(t *testing.T) {
|
||||
key, _ := mwc.WantKey("ace", "web")
|
||||
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
|
||||
for _, c := range []struct {
|
||||
name, node, module, why, by string
|
||||
refuse error
|
||||
dropped bool
|
||||
}{
|
||||
{"dropped", "ace", "web", "the clean state", "person.jochen", nil, true},
|
||||
{"no want there", "ace", "db", "the clean state", "person.jochen", nil, false},
|
||||
{"no why", "ace", "web", "", "person.jochen", nil, false},
|
||||
{"no caller", "ace", "web", "the clean state", "", nil, false},
|
||||
{"not recorded", "ace", "web", "the clean state", "person.jochen", errors.New("no responders"), false},
|
||||
} {
|
||||
st := memStore{wantsState: &memKV{values: map[string][]byte{key: held}}}
|
||||
var asked []string
|
||||
ask := func(verb string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, verb+" "+string(raw))
|
||||
return json.RawMessage(`{}`), c.refuse
|
||||
}
|
||||
out, err := dropWant(st, ask, c.node, c.module, c.why, c.by)
|
||||
_, there := st[wantsState].values[key]
|
||||
switch {
|
||||
case c.dropped && (err != nil || there || len(asked) != 1 || !strings.Contains(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], wantDropTool+" "+key)):
|
||||
t.Errorf("%s: %v, %v; the want left %v, asked %v", c.name, out, err, there, asked)
|
||||
case !c.dropped && (err == nil || !there):
|
||||
t.Errorf("%s: %v, %v; the want left %v, want it refused and kept", c.name, out, err, there)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-121 (2), H-A-123: assigner_want_drop's all form drops every want key, held or not, and every value of an earlier form,
|
||||
// each recorded as a hand act first, answering the count per kind; a held fingerprint's key stays; a drop not recorded stops it;
|
||||
// the all form names no node or module.
|
||||
func TestEveryWantIsDroppedByHandAndCounted(t *testing.T) {
|
||||
heldKey, _ := mwc.WantKey("ace", "web")
|
||||
curKey, _ := mwc.WantKey("ace", "db")
|
||||
nodeKey, _ := mwc.WantNodeKey("ace")
|
||||
oldKey, _ := mwc.WantKey("g14", "web")
|
||||
fpKey, _ := mwc.HeldKey("web", fpTest)
|
||||
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
|
||||
cur, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest})
|
||||
node, _ := mwc.EncodeNodeWant(mwc.NodeWant{Self: true})
|
||||
values := func() map[string][]byte {
|
||||
return map[string][]byte{heldKey: held, curKey: cur, nodeKey: node, oldKey: []byte(`{"fingerprint":"old"}`), fpKey: []byte(`{}`)}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
refuse error
|
||||
left int
|
||||
}{
|
||||
{"dropped", nil, 1},
|
||||
{"not recorded", errors.New("no responders"), 5},
|
||||
} {
|
||||
st := memStore{wantsState: &memKV{values: values()}}
|
||||
var asked []string
|
||||
ask := func(verb string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, string(raw))
|
||||
return json.RawMessage(`{}`), c.refuse
|
||||
}
|
||||
out, err := dropAllWants(st, ask, "the clean state", "person.jochen")
|
||||
if left := len(st[wantsState].values); left != c.left || (c.refuse == nil) != (err == nil) {
|
||||
t.Errorf("%s: %v, %v; %d keys left, want %d", c.name, out, err, left, c.left)
|
||||
}
|
||||
if c.refuse != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := st[wantsState].values[fpKey]; !ok {
|
||||
t.Errorf("the held fingerprint's key %s is dropped: it is no want", fpKey)
|
||||
}
|
||||
counted := out.(map[string]any)["counted"].(map[string]int)
|
||||
if counted[dropHeld] != 1 || counted[dropCurrent] != 2 || counted[dropUndecodable] != 1 || len(asked) != 4 {
|
||||
t.Errorf("counted %v, asked %d hand acts; want held 1, current 2, undecodable 1 and 4 acts", counted, len(asked))
|
||||
}
|
||||
}
|
||||
if _, err := dropAllWants(memStore{wantsState: &memKV{values: values()}}, nil, "", "person.jochen"); err == nil {
|
||||
t.Error("the all form dropped with no why")
|
||||
}
|
||||
run := wantDropTools(memStore{wantsState: &memKV{values: values()}}, nil)[0].Run
|
||||
if _, err := run(map[string]any{"all": "true", "node": "ace", "why": "x"}); err == nil || !strings.Contains(err.Error(), "name no node") {
|
||||
t.Errorf("the all form with a node: %v, want it refused", err)
|
||||
}
|
||||
}
|
||||
@@ -71,21 +71,15 @@ func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
return s.direct(state)
|
||||
}
|
||||
|
||||
// direct reads a state key by key, for a small state of the seat's own.
|
||||
// direct reads a state whole, in pages where the runtime has them (readAll).
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
all, err := readAll(s.st, state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
out := make(map[string][]byte, len(all))
|
||||
for k, v := range all {
|
||||
out[k] = v.raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// Why a want is what it is (lead-47 2026-11-15, the tools rule: novox's want of the indexer stood unwritten for a day and nothing
|
||||
// said why). Each pass writes, in the state written beside the pipeline ends, one key `why.<node>.<module>`: the step of the pass
|
||||
// that decided the want and what it read there. A key whose time stands still says the pass no longer judges that module.
|
||||
// assigner_why reads them.
|
||||
|
||||
// whyPrefix starts every key of a want's why in the state written; a pipeline end's key starts with a module's name.
|
||||
const whyPrefix = "why."
|
||||
|
||||
// whyTool is the tool that reads them.
|
||||
const whyTool = "assigner_why"
|
||||
|
||||
// The steps of the pass that decide a want.
|
||||
const (
|
||||
whyNoPipeline = "no-passed-pipeline" // no version of the module released on the branch passed its pipeline: the want written stays
|
||||
whyPassed = "passed-pipeline" // the want names the newest version released on the branch whose pipeline passed (design 47 H §750)
|
||||
)
|
||||
|
||||
// whyStep is the step that decided a want: a passed pipeline's version, or none yet.
|
||||
func whyStep(passed bool) string {
|
||||
if passed {
|
||||
return whyPassed
|
||||
}
|
||||
return whyNoPipeline
|
||||
}
|
||||
|
||||
// wantWhy is one key's value.
|
||||
type wantWhy struct {
|
||||
At time.Time `json:"at"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// Fp, Version and Pipeline are the build the want names and the pipeline whose passed end released it (design 47 H §750).
|
||||
Fp string `json:"fp,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Step string `json:"step"`
|
||||
// Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before).
|
||||
Before *mwc.Want `json:"before,omitempty"`
|
||||
After *mwc.Want `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
func whyKey(node, module string) string { return whyPrefix + node + "." + module }
|
||||
|
||||
// sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear.
|
||||
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
one := w
|
||||
one.At = time.Now().UTC()
|
||||
if b, ok := before[node]; ok {
|
||||
one.Before = &b
|
||||
}
|
||||
if out != nil {
|
||||
if a, ok := out.Wants[node]; ok {
|
||||
one.After = &a
|
||||
}
|
||||
}
|
||||
same := one
|
||||
same.At = time.Time{}
|
||||
cmp, _ := json.Marshal(same)
|
||||
key := whyKey(node, module)
|
||||
if p.whys[key] == string(cmp) {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(one)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, raw); err != nil {
|
||||
p.logf("the why of %s on %s is not written: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
p.whys[key] = string(cmp)
|
||||
}
|
||||
}
|
||||
|
||||
// whyTools is the tool assigner_why: every want's why, or those of one module or one node.
|
||||
func whyTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline yet, or a passed pipeline's version), the build, version and pipeline it names, the want before and the one written. A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
|
||||
Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
module, node := arg(args, "module"), arg(args, "node")
|
||||
sort.Strings(ks)
|
||||
out := map[string]wantWhy{}
|
||||
for _, k := range ks {
|
||||
rest, ok := strings.CutPrefix(k, whyPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
n, mod, _ := strings.Cut(rest, ".")
|
||||
if (module != "" && mod != module) || (node != "" && n != node) {
|
||||
continue
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
var w wantWhy
|
||||
if json.Unmarshal(raw, &w) == nil {
|
||||
out[rest] = w
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
|
||||
// sayPass writes, under `why.pass`, whether the last pass judged the wants at all, once per change of its words.
|
||||
func (p *passer) sayPass(ctx context.Context, words string) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
if p.whys[whyPrefix+"pass"] == words {
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(wantWhy{At: time.Now().UTC(), Step: words})
|
||||
if _, err := kv.Put(ctx, whyPrefix+"pass", raw); err == nil {
|
||||
p.whys[whyPrefix+"pass"] = words
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -43,12 +42,8 @@ type world struct {
|
||||
ends builtEnds
|
||||
// released are the catalog's version-released heard, by module and source revision (buildended.go).
|
||||
released releasedVersions
|
||||
// delivered are the assigner's own version-delivered heard and said (delivered.go).
|
||||
delivered deliveredVersions
|
||||
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
|
||||
edges map[string]map[string][]meshcatalog.Edge
|
||||
// taken is the newest source revision of each watched branch the indexer's take-in.taken named, as heard (orderwaits.go, §298, A-260).
|
||||
taken takenSources
|
||||
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
|
||||
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
assignmentKeys *mmac.Keys
|
||||
@@ -139,23 +134,6 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
|
||||
func (w *world) lastHealthy(node, module, fp string) string {
|
||||
key, err := nodeengineclient.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
list := w.healthy[key]
|
||||
for i := len(list) - 1; i >= 0; i-- {
|
||||
if list[i] != fp {
|
||||
return list[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// read is a copy of what a bucket holds now.
|
||||
func (w *world) read(bucket string) map[string][]byte {
|
||||
w.mu.Lock()
|
||||
@@ -414,11 +392,7 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNode {
|
||||
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
|
||||
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
n.Reads[node] = h.Record.ManifestReads
|
||||
}
|
||||
if isNode { // what the node's engine reads is the state engine-reads' alone (design 47 H-A-104)
|
||||
continue
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHave(value); err == nil {
|
||||
@@ -464,37 +438,6 @@ func (w *world) has(bucket, key string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181).
|
||||
func (w *world) built(module, fp string) bool {
|
||||
e, ok := w.ends.endOf(module, fp)
|
||||
return ok && e.Outcome == builds.BuildDone
|
||||
}
|
||||
|
||||
// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G
|
||||
// §575, A-162): the words of a hold under rule 1.
|
||||
func (w *world) buildStanding(module, fp string) (failed bool, words string) {
|
||||
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed {
|
||||
return true, e.Error
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR
|
||||
// 0618 §1, A-181): nil when it is not done or names none.
|
||||
func (w *world) buildNeeds(module, fp string) []string {
|
||||
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone {
|
||||
return e.Needs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
|
||||
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
|
||||
func (w *world) recordReads(node string) ([]string, bool) {
|
||||
r, ok := w.records()[node]
|
||||
return r.ManifestReads, ok
|
||||
}
|
||||
|
||||
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
|
||||
func (w *world) knownNodes() []string {
|
||||
set := map[string]bool{}
|
||||
@@ -610,3 +553,18 @@ func (w *world) placedBranches() map[string]map[string]string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedOn is the modules the assignments place on node.
|
||||
func (w *world) assignedOn(node string) []string {
|
||||
var out []string
|
||||
for _, m := range w.assignments()[node].Modules {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignmentOf is node's assignments, or false where none is held.
|
||||
func (w *world) assignmentOf(node string) (mma.Assignments, bool) {
|
||||
a, ok := w.assignments()[node]
|
||||
return a, ok
|
||||
}
|
||||
|
||||
@@ -89,16 +89,13 @@ replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate =>
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient => ../mesh-delivery-contracts/go/clients/meshdeliverycheckerclient
|
||||
|
||||
require git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
|
||||
)
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
// Package acting is the assigner's wait for an acting part's new version (design 47 G §266, §268, §276): an entry of the assignment
|
||||
// graph whose edge of kind acting names a step the assigner holds (composing, placing) waits until the acting part's version runs
|
||||
// on the node of the edge, with the bound of the rule that gave the edge, or the step's default. The package holds the rules and
|
||||
// the kept record; the assigner's process reads what runs, says the events and keeps the records in its state `waits`.
|
||||
package acting
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// HowNow is the `how` of assigner_release for a wait for an acting part: the entry is wanted now, with the version running.
|
||||
const HowNow = "now"
|
||||
|
||||
// The ways a wait ends, as `wait-ended` says them.
|
||||
const (
|
||||
HowRan = "ran"
|
||||
HowReleased = "released"
|
||||
HowReplaced = "replaced"
|
||||
)
|
||||
|
||||
// Prefix opens the key of an acting wait in the state, apart from the outside waits' `<node>.<module>.<fingerprint hex>`.
|
||||
const Prefix = "acting."
|
||||
|
||||
// OrderPrefix opens the key of an order wait (design 47 G §298, A-259): a want held for the order of a delivery that crosses a form change,
|
||||
// kept beside the acting waits in the state `waits`, as `order.<node>.<module>.<part>`.
|
||||
const OrderPrefix = "order."
|
||||
|
||||
// The three order edges of §298, as a kept wait's Order names them.
|
||||
const (
|
||||
// OrderBackup: the new versions of the node-backup holder and of the holder of the bus module's seat run before a module whose kept
|
||||
// data's form changes is moved.
|
||||
OrderBackup = "backup-and-broker-first"
|
||||
// OrderEngineLast: at a revert a node's node-engine goes back last on that node.
|
||||
OrderEngineLast = "node-engine-last"
|
||||
// OrderTakenIn: the reverted assigner and gate are wanted only once the reverted indexer took in the trunk's head.
|
||||
OrderTakenIn = "taken-in-first"
|
||||
)
|
||||
|
||||
// Held says whether the assigner holds the entries of step: composing and placing (§276's table).
|
||||
func Held(step string) bool { return step == seatdef.StepCompose || step == seatdef.StepPlace }
|
||||
|
||||
// Wait is one wait as kept: Module's entry on Node waits for Part's version Fp to run on PartNode, in Step.
|
||||
type Wait struct {
|
||||
V int `json:"v"`
|
||||
Step string `json:"step"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Part string `json:"part"`
|
||||
PartNode string `json:"part-node"`
|
||||
Fp string `json:"fp"`
|
||||
Entry string `json:"entry"`
|
||||
Since time.Time `json:"since"`
|
||||
ClockFrom time.Time `json:"clock-from"`
|
||||
Bound string `json:"bound"`
|
||||
// Released is a release by hand: the entry moves on with the part's old version; EndSaid says wait-ended was said for it.
|
||||
// Order is which of §298's edges the wait stands on; empty for an acting wait.
|
||||
Order string `json:"order,omitempty"`
|
||||
Released bool `json:"released,omitempty"`
|
||||
EndSaid bool `json:"end-said,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// Key is the key of a wait in the state.
|
||||
func Key(node, module, part string) string { return Prefix + node + "." + module + "." + part }
|
||||
|
||||
// OrderKey is the key of an order wait in the state.
|
||||
func OrderKey(node, module, part string) string {
|
||||
return OrderPrefix + node + "." + module + "." + part
|
||||
}
|
||||
|
||||
// IsOrder says a kept value is an order wait's record.
|
||||
func IsOrder(key string) bool { return strings.HasPrefix(key, OrderPrefix) }
|
||||
|
||||
// Is says a kept value is an acting wait's record.
|
||||
func Is(key string) bool { return strings.HasPrefix(key, Prefix) }
|
||||
|
||||
// Encode is w as JSON.
|
||||
func Encode(w Wait) ([]byte, error) { w.V = 1; return json.Marshal(w) }
|
||||
|
||||
// Decode reads a kept wait.
|
||||
func Decode(raw []byte) (Wait, error) {
|
||||
var w Wait
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
return w, err
|
||||
}
|
||||
if w.V != 1 || w.Step == "" || w.Node == "" || w.Module == "" || w.Part == "" {
|
||||
return w, fmt.Errorf("acting: %q is not a kept acting wait", raw)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// BoundOf is the bound of the edge's wait: its rule's, else its step's default (seatdef.ActingRule.Wait).
|
||||
func BoundOf(e meshwants.Edge) time.Duration {
|
||||
return seatdef.ActingRule{Step: e.Step, Bound: e.Bound}.Wait()
|
||||
}
|
||||
|
||||
// Pending are the acting edges of the assigner's steps whose part's version does not run yet: runs says the edge's part runs
|
||||
// its version on the edge's node.
|
||||
func Pending(edges []meshwants.Edge, runs func(e meshwants.Edge) bool) []meshwants.Edge {
|
||||
var out []meshwants.Edge
|
||||
for _, e := range edges {
|
||||
if e.Kind == "acting" && Held(e.Step) && !runs(e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Body is the wait as the seat mesh-wants says it.
|
||||
func Body(w Wait, owner string) meshwants.Wait {
|
||||
since, from := w.Since.UTC(), w.ClockFrom.UTC()
|
||||
return meshwants.Wait{Step: w.Step, Owner: owner, Node: w.Node, Module: w.Module, Part: w.Part, Fp: w.Fp, Since: since, ClockFrom: &from, Bound: w.Bound,
|
||||
Words: words(w)}
|
||||
}
|
||||
|
||||
func words(w Wait) string {
|
||||
switch w.Order {
|
||||
case OrderBackup:
|
||||
return fmt.Sprintf("%s on %s moves its kept data across a form change: it waits in %s for %s at %s to run on %s, which the restore point needs", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
|
||||
case OrderEngineLast:
|
||||
return fmt.Sprintf("%s on %s goes back last on its node: it waits in %s for %s at %s to run on %s, restored", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
|
||||
case OrderTakenIn:
|
||||
return fmt.Sprintf("%s on %s goes back only once the indexer took in the trunk's head: it waits in %s for source revision %s to be taken in", w.Module, w.Node, w.Step, w.Fp)
|
||||
}
|
||||
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
|
||||
}
|
||||
|
||||
func short(fp string) string {
|
||||
if len(fp) > 19 {
|
||||
return fp[:19]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
@@ -13,6 +14,10 @@ import (
|
||||
|
||||
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
|
||||
func assignerAsk(asking int, a oc.Ask) asks.Ask {
|
||||
a.Headline, a.Explanation, a.Details, a.OnExpiry = said(a.Headline), said(a.Explanation), said(a.Details), said(a.OnExpiry)
|
||||
for i := range a.Options {
|
||||
a.Options[i].Label, a.Options[i].Does = said(a.Options[i].Label), said(a.Options[i].Does)
|
||||
}
|
||||
raw, err := json.Marshal(a)
|
||||
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
|
||||
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
|
||||
@@ -20,6 +25,17 @@ func assignerAsk(asking int, a oc.Ask) asks.Ask {
|
||||
return asks.Ask{Asking: asking, Ask: raw}
|
||||
}
|
||||
|
||||
// longHex is a digest or fingerprint whole: 64 hexadecimal digits.
|
||||
var longHex = regexp.MustCompile(`[0-9a-f]{64}`)
|
||||
|
||||
// said is words as an ask carries them out of the mesh: each digest or fingerprint cut to its first 12 digits, which the
|
||||
// router lets pass, where it refuses a long hexadecimal string as a secret's shape (mesh-sdk outward; novox 2026-11-16 01:24:
|
||||
// every backup ask was refused for its "(composition sha256:…)" and the fingerprints in the node's words, and the operator
|
||||
// was never asked).
|
||||
func said(words string) string {
|
||||
return longHex.ReplaceAllStringFunc(words, func(h string) string { return h[:12] })
|
||||
}
|
||||
|
||||
// AskOf is the router's ask an asks-state value carries.
|
||||
func AskOf(a asks.Ask) (oc.Ask, error) {
|
||||
var ask oc.Ask
|
||||
|
||||
@@ -3,7 +3,6 @@ package assign
|
||||
import (
|
||||
"fmt"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -16,56 +15,12 @@ import (
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
|
||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Options: []oc.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
OnExpiry: "the version stays held",
|
||||
About: key,
|
||||
})
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
const (
|
||||
OptionRestore = "restore"
|
||||
OptionLeaveStopped = "leave-stopped"
|
||||
)
|
||||
|
||||
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's
|
||||
// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the
|
||||
// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
})
|
||||
}
|
||||
|
||||
// The options of the outside ask (design 47 G §706).
|
||||
const (
|
||||
OptionGoOn = "go-on"
|
||||
@@ -106,92 +61,6 @@ func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observ
|
||||
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
|
||||
}
|
||||
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
|
||||
key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
|
||||
}
|
||||
c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = occ.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
|
||||
key := assignment.Module() + ".failed-alone." + node + "." + module
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > occ.HeadlineLength {
|
||||
headline = string([]rune(headline)[:occ.HeadlineLength])
|
||||
}
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
|
||||
key := assignment.Module() + ".neither." + node + "." + module
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > occ.HeadlineLength {
|
||||
headline = string([]rune(headline)[:occ.HeadlineLength])
|
||||
}
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) oc.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) oc.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
@@ -288,22 +157,6 @@ func NextBackupAsking(after int) int {
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a asks.Ask) string {
|
||||
ask, err := AskOf(a)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
@@ -311,14 +164,14 @@ func BackupAnswerWords(option string, asking int) string {
|
||||
}
|
||||
|
||||
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
|
||||
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
|
||||
module, short(fp), node, words, module),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -22,11 +23,10 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its key, of a kind the view may answer.
|
||||
func TestTheHoldAskIsValid(t *testing.T) {
|
||||
t.Run("hold", theHoldAskIsValid)
|
||||
t.Run("restore", theRestoreAskIsValid)
|
||||
// 1297/B9 (the assigner's half): the backup ask is a valid ask of the router and of mesh-sdk's layout at its key, of a kind the
|
||||
// view may answer; the hold and restore asks went with the hold of a fingerprint (design 47 H §750, H-A-55).
|
||||
func TestTheAssignersAsks(t *testing.T) {
|
||||
t.Run("backup", theBackupAskIsValid)
|
||||
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
|
||||
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
|
||||
}
|
||||
@@ -39,8 +39,6 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
|
||||
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
|
||||
now := asked.Add(8 * 24 * time.Hour)
|
||||
digest := "sha256:" + strings.Repeat("e", 64)
|
||||
holdKey, _ := asks.HoldAskKey("nats", fNew)
|
||||
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
|
||||
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
|
||||
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
|
||||
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
|
||||
@@ -62,9 +60,7 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
|
||||
put *puts.Put
|
||||
why string // "" when it is written again
|
||||
}{
|
||||
{name: "hold", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeExpired},
|
||||
{name: "restore", key: restoreKey, a: ptr(RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeCancelled},
|
||||
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, digest, "w", 0, asked)), outcome: occ.OutcomeCancelled},
|
||||
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeCancelled},
|
||||
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
|
||||
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
|
||||
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
|
||||
@@ -72,11 +68,11 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
|
||||
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
|
||||
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
|
||||
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
|
||||
{name: "another asking stands", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(now), now)),
|
||||
ended: meshdeliveryclient.AskingID(holdKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
|
||||
{name: "taken away", key: holdKey, ended: "hold-x", outcome: occ.OutcomeExpired, why: "taken away"},
|
||||
{name: "refused", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
|
||||
{name: "replaced", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
|
||||
{name: "another asking stands", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", AskingSeconds(now), now)),
|
||||
ended: meshdeliveryclient.AskingID(backupKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
|
||||
{name: "taken away", key: backupKey, ended: "backup-x", outcome: occ.OutcomeExpired, why: "taken away"},
|
||||
{name: "refused", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
|
||||
{name: "replaced", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var current []byte
|
||||
@@ -133,12 +129,12 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
|
||||
// answered or another asker's ask, and one the key no longer holds, are not.
|
||||
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
|
||||
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
|
||||
holdKey, _ := asks.HoldAskKey("nats", fNew)
|
||||
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
|
||||
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
|
||||
hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)
|
||||
restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
|
||||
backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking
|
||||
holdKey, _ := asks.BackupAskKey("g14", "nats", fNew)
|
||||
restoreKey, _ := asks.BackupAskKey("ace", "nats", fNew)
|
||||
hold := BackupAsk(holdKey, "g14", "nats", fNew, "w", AskingSeconds(asked), asked)
|
||||
restore := BackupAsk(restoreKey, "ace", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
|
||||
backup := BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked.Add(-time.Hour)) // no asking
|
||||
held := map[string][]byte{}
|
||||
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
|
||||
v, err := EncodeAsk(key, a)
|
||||
@@ -173,27 +169,27 @@ func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
|
||||
|
||||
func ptr(a asks.Ask) *asks.Ask { return &a }
|
||||
|
||||
func theHoldAskIsValid(t *testing.T) {
|
||||
key, err := asks.HoldAskKey("mail", fNew)
|
||||
func theBackupAskIsValid(t *testing.T) {
|
||||
key, err := asks.BackupAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
a := BackupAsk(key, "ace", "mail", fNew, "health failed, phase running: connection refused", 0, now)
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
// The ask names the machine whose backup failed and the backup's words.
|
||||
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
|
||||
}
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
// An ask written again is another asking, under an id of its own.
|
||||
again := BackupAsk(key, "ace", "mail", fNew, "x", 7, now)
|
||||
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
|
||||
t.Errorf("the asking again = %d %s", again.Asking, mustAsk(t, again).ID)
|
||||
}
|
||||
if _, err := EncodeAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
@@ -203,7 +199,7 @@ func theHoldAskIsValid(t *testing.T) {
|
||||
}
|
||||
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
|
||||
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
|
||||
good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0))
|
||||
good := BackupAsk(key, "ace", "mail", fNew, "x", 0, time.Unix(100, 0))
|
||||
other := good
|
||||
other.Asking = 3 // its id is the asking 0's
|
||||
empty := good
|
||||
@@ -218,68 +214,11 @@ func theHoldAskIsValid(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
|
||||
func theRestoreAskIsValid(t *testing.T) {
|
||||
key, err := asks.RestoreAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", 2, now)
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
if _, err := viewstate.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range mustAsk(t, a).Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
// 1952/WM6: the assigner's conditions are findings.
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
@@ -290,9 +229,6 @@ func saidConditionsAreFindings(t *testing.T) {
|
||||
raised, cleared oc.Condition
|
||||
key string
|
||||
}{
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
@@ -335,8 +271,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, "no space left", 0, now)
|
||||
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -359,15 +294,17 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", mustAsk(t, b))
|
||||
}
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
|
||||
// the id binds the ask to its number (review round 1, item 4); its key binds it to the build (design 47 H §750: a want names no composition)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
// novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole.
|
||||
held := BackupAsk(bkey, "novox", "mail", fNew, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
|
||||
if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) {
|
||||
t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
@@ -395,6 +332,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
@@ -439,22 +377,6 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// mustAsk is the router's ask an asks-state value carries.
|
||||
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
|
||||
t.Helper()
|
||||
|
||||
@@ -154,18 +154,3 @@ func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
|
||||
// KindActingCycle is the finding of a want held on a cycle of acting waits no rule breaks (design 47 G A-323, A-330): scope module,
|
||||
// id `<node>.<module>` of the held want, naming the cycle with each edge's step; cleared by the next pass that finds it held on no
|
||||
// cycle.
|
||||
const KindActingCycle = "acting-cycle"
|
||||
|
||||
// ActingCycleFinding is the finding acting-cycle of module's want on node, held on cycle.
|
||||
func ActingCycleFinding(node, module, cycle string) Observed {
|
||||
return Observed{Scope: mcc.ScopeModule, ID: node + "." + module, Kind: KindActingCycle, Machine: node, Severity: "warning",
|
||||
Summary: fmt.Sprintf("the want of %s on %s is held on a cycle of acting waits: %s", module, node, cycle),
|
||||
Headline: fmt.Sprintf("%s on %s waits on a cycle of acting parts", module, node),
|
||||
Explanation: fmt.Sprintf("Each acting part on the cycle %s waits for another's new version to run, so none moves by itself. "+
|
||||
"assigner_release lets one move with the version running.", cycle),
|
||||
Source: "assigner." + KindActingCycle}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,11 @@ import (
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
@@ -48,66 +46,15 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
|
||||
const KindProvisionFailed = "provision-failed"
|
||||
|
||||
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
|
||||
func ProvisionFailedKey(key string) string {
|
||||
return mcc.Key(mcc.ScopeProvider, key, KindProvisionFailed)
|
||||
}
|
||||
|
||||
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
|
||||
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) {
|
||||
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
|
||||
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
|
||||
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
|
||||
Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
|
||||
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
|
||||
Source: "assigner." + KindProvisionFailed}, at)
|
||||
}
|
||||
|
||||
// encode is the observation as the assigner's finding at at.
|
||||
func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil }
|
||||
|
||||
func also(provider, consumer string) []string {
|
||||
if provider == "" || provider == consumer {
|
||||
return nil
|
||||
}
|
||||
return []string{provider}
|
||||
}
|
||||
|
||||
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
|
||||
const KindProvisionMissing = "provision-missing"
|
||||
|
||||
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
|
||||
func ProvisionMissingKey(key string) string {
|
||||
return mcc.Key(mcc.ScopeProvider, key, KindProvisionMissing)
|
||||
}
|
||||
|
||||
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
|
||||
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) {
|
||||
parts := strings.SplitN(key, ".", 3)
|
||||
if len(parts) != 3 {
|
||||
return mcseat.Finding{}, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
|
||||
}
|
||||
node, module, provision := parts[0], parts[1], parts[2]
|
||||
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
|
||||
"no provider of it may be placed", module, node, provision, bound, module)
|
||||
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
|
||||
Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
|
||||
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
|
||||
Source: "assigner." + KindProvisionMissing}, at)
|
||||
}
|
||||
|
||||
// KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6).
|
||||
const KindUnplaced = "unplaced"
|
||||
|
||||
|
||||
@@ -34,6 +34,9 @@ type PutWorld struct {
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Version is the version released at the put's fingerprint, matched by its build's parts as the pass matches a mover's;
|
||||
// "" when none is released (design 47 H2 H-A-490 addendum).
|
||||
Version string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *ComposedDigest
|
||||
@@ -79,6 +82,26 @@ func StepPut(w PutWorld) PutNext {
|
||||
if !w.Assigned {
|
||||
return say(puts.Refused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its assignments first)", w.Module, w.Node)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding && w.Build == "" {
|
||||
// Only where the builder's store holds no build of it: one it built goes the put's whole way (composition, the yes).
|
||||
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP {
|
||||
// The build the node runs now (design 47 G A-427): read from the node's running statement, held there with the
|
||||
// composition its have names, and the builder's store is not asked for it (a part started by hand has none there).
|
||||
if h := w.Have; h == nil || h.Fp != p.FP || h.Composed == "" {
|
||||
return say(puts.Refused, "%s runs %s at %s, and its have names no composition at that fingerprint, so the want cannot name what runs", w.Node, w.Module, fp)
|
||||
}
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Composed, p.Version = w.Have.Composed, w.Version
|
||||
words := fmt.Sprintf("the want of %s on %s names %s, the build it runs now (composition %s), from its running statement; the builder's store is not asked",
|
||||
w.Module, w.Node, fp, short(p.Composed))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
return PutNext{Put: applied(w, p.Said(w.Now, puts.Wanted, words))}
|
||||
}
|
||||
}
|
||||
switch w.Build {
|
||||
case builds.BuildDone:
|
||||
case builds.BuildFailed:
|
||||
@@ -86,13 +109,26 @@ func StepPut(w PutWorld) PutNext {
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put; the build the node runs now
|
||||
// can (A-427), and the refusal names it.
|
||||
runs := "states no running build of it"
|
||||
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" {
|
||||
runs = "runs " + short(r.Fingerprint)
|
||||
}
|
||||
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s), and %s %s by its running statement: put that, or a version the builder built", w.Module, fp, short(w.Newest), w.Node, runs)
|
||||
}
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding {
|
||||
// The want names the build's released version beside its fingerprint, which the node-engine composes from (H-A-490
|
||||
// addendum): a build with none released is refused, never wanted by its fingerprint alone.
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Version = w.Version
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
@@ -190,9 +226,10 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
|
||||
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and the version released at it, marked put, and held when
|
||||
// it holds.
|
||||
func PutWant(p puts.Put) mwc.Want {
|
||||
return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
return mwc.Want{Fp: p.FP, Version: p.Version, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// KindAssignmentUnlisted is a module assigned to a node that no watched branch lists, left out of that node's composition
|
||||
// (design 47 H-A-148, H-A-149): scope delivery, id `<node>.<module>`, severity warning.
|
||||
const (
|
||||
KindAssignmentUnlisted = "assignment-unlisted"
|
||||
ScopeAssignmentUnlisted = "delivery"
|
||||
)
|
||||
|
||||
// UnlistedKey is its finding's key, `delivery.<node>.<module>.assignment-unlisted`.
|
||||
func UnlistedKey(node, module string) string {
|
||||
return views.FindingKey(ScopeAssignmentUnlisted, KeyID(node+"."+module), KindAssignmentUnlisted)
|
||||
}
|
||||
|
||||
// UnlistedOfKey is the node and module of an assignment-unlisted finding's key, or false for another finding's.
|
||||
func UnlistedOfKey(key string) (node, module string, ok bool) {
|
||||
rest, ok := strings.CutPrefix(key, ScopeAssignmentUnlisted+".")
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
if rest, ok = strings.CutSuffix(rest, "."+KindAssignmentUnlisted); !ok {
|
||||
return "", "", false
|
||||
}
|
||||
node, module, ok = strings.Cut(rest, ".")
|
||||
return node, module, ok && node != "" && module != "" && !strings.Contains(module, ".")
|
||||
}
|
||||
|
||||
// UnlistedFinding is the finding of module assigned to node and listed on none of its watched branches, raised by the composition
|
||||
// step that leaves the assignment out (H-A-149).
|
||||
func UnlistedFinding(node, module string, at time.Time) (mcseat.Finding, error) {
|
||||
summary := fmt.Sprintf("%s is assigned %s, which no watched branch lists, so its composition leaves the assignment out", node, module)
|
||||
return encode(Observed{Scope: ScopeAssignmentUnlisted, ID: node + "." + module, Kind: KindAssignmentUnlisted, Machine: node,
|
||||
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: clipHeadline(module + " on " + node + " is listed nowhere"),
|
||||
Explanation: summary + ": a stale assignment is a fault of the assignments, not of the change judged (ADR 0578 §2). " +
|
||||
"Take it off, or register the module on a watched branch.",
|
||||
Source: "assigner." + KindAssignmentUnlisted}, at)
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
)
|
||||
|
||||
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
|
||||
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
|
||||
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
|
||||
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
|
||||
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
|
||||
func (m Module) verbWaiting(node string) (string, bool) {
|
||||
if m.Served == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
|
||||
if !m.Served(v) {
|
||||
return graph.VerbWaitWords(v, m.Name, node), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
|
||||
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
|
||||
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
|
||||
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
|
||||
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
|
||||
func (m Module) busStateWaiting(node string) (string, bool) {
|
||||
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
|
||||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
|
||||
|
||||
return "", false
|
||||
}
|
||||
return graph.BusStateWaitWords(m.Name, node), true
|
||||
}
|
||||
@@ -2,359 +2,79 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||
// A want names the newest released version whose pipeline passed (design 47 H §750): for every node whose assignment places
|
||||
// a module on a watched branch, the want names the version, the pipeline and the mover's rank in it, written tier by tier
|
||||
// (§751). It is held for nothing else: the rules of ADR 0429 §6 (the build done, not held, the canary and step, every
|
||||
// provider healthy, the grant, the composition and the reader capabilities, the provisions, the served verbs and the bus
|
||||
// state), the network hold and A-428's exceptions end (H-A-55). A put owns its node's want as before (ADR 0546), and the
|
||||
// operator's yes on a module of policy record is the node-engine's have and the asker's (the pass's yeses).
|
||||
|
||||
// Target is the build a node's want names: its fingerprint and released version, with the pipeline whose passed end released
|
||||
// it and the mover's rank in that pipeline; a placement's build names no pipeline and no rank (design 47 H §750, H-A-37).
|
||||
type Target struct {
|
||||
Fp string
|
||||
Version string
|
||||
Pipeline string
|
||||
Rank int64
|
||||
}
|
||||
|
||||
// Module is what the want rule reads of one module: the nodes it is assigned to, the build each node's want names, the puts
|
||||
// that own a node's want and the nodes that compose it themselves.
|
||||
type Module struct {
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
// BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole
|
||||
// (design 47 G §575, A-162): the words of a hold under rule 1.
|
||||
BuildFailed bool
|
||||
BuildError string
|
||||
Held bool
|
||||
Granted bool
|
||||
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
||||
// that runs an older one names it (issue 1945). Nil: none.
|
||||
GrantedAt func(fp string) bool
|
||||
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
||||
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
|
||||
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
|
||||
Provisions func(node string) map[string]meshprovisionerclient.Provision
|
||||
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
||||
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
||||
Answering func(provision string) []catalogue.Manifest
|
||||
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
|
||||
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
|
||||
// is waited on for want of an offer.
|
||||
Offered func(provision string) bool
|
||||
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
|
||||
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
|
||||
// served (Served). Nil: no graph node is read, and no want waits on a verb.
|
||||
Invokes []string
|
||||
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
|
||||
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
|
||||
Served func(invoke string) bool
|
||||
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
|
||||
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
|
||||
// runs, and no want waits on bus state.
|
||||
Offering func(provision string) bool
|
||||
Name string
|
||||
Manifest catalogue.Manifest
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (ComposedDigest, bool)
|
||||
// Target is the build node's want names: the newest version of the module released on the branch the node's placement
|
||||
// names whose pipeline passed and whose tier is written (§750, §751), or the placement's build (H-A-5); false: none yet,
|
||||
// and the want written stays.
|
||||
Target func(node string) (Target, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||
Restored func(node string) bool
|
||||
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rule passes over it.
|
||||
Puts map[string]puts.Put
|
||||
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
|
||||
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
|
||||
Needs []string
|
||||
// FormNeeds are the reference forms the module's build at FP is written in (`form:<name>`, the builder's build key's
|
||||
// needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the
|
||||
// build names none, and holds nothing.
|
||||
FormNeeds []string
|
||||
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the assignments (issue 1126). Nil: no node does.
|
||||
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). Nil: no
|
||||
// node does.
|
||||
Self func(node string) bool
|
||||
// KeptBuilds answers what a want at a fingerprint names of its build: its parts and its keeping node as the builder's `build.ended`
|
||||
// names them, or the released version and parts of the catalog's `version-released` (design 47 G §379, §526, A-96), for a node that states the capability WantParts. Nil: none is read.
|
||||
KeptBuilds func(fp string) (KeptBuild, bool)
|
||||
// MachineHeld says whether the node's own network condition is open, which holds the machine as a whole: no want moves to it
|
||||
// until that clears (design 47 G §706 *M's own network*). Nil: none is held.
|
||||
MachineHeld func(node string) bool
|
||||
// Acting are the acting edges of the assignment graph entry of this module on node that the assigner holds (composing, placing)
|
||||
// and whose wait was not released by hand (design 47 G §266, §268): the want moves only once each acting part's version runs
|
||||
// on the edge's node, as Nodes.Running says it. Nil: none.
|
||||
Acting func(node string) []ActingEdge
|
||||
}
|
||||
|
||||
// ActingEdge is one acting edge: Part's version Fp acts in Step on Node, before the entry moves.
|
||||
type ActingEdge struct{ Node, Part, Fp, Step string }
|
||||
|
||||
// Runs says the part's version of the edge runs on its node, as the node-engine seat's `running` says it.
|
||||
func (n Nodes) Runs(e ActingEdge) bool {
|
||||
if n.Running != nil {
|
||||
r, ok := n.Running[e.Node][e.Part]
|
||||
return ok && !r.Stopped && r.Fingerprint == e.Fp
|
||||
}
|
||||
h, ok := n.Haves[e.Node][e.Part]
|
||||
return ok && healthy(h, e.Fp)
|
||||
}
|
||||
|
||||
// actingWaiting says why the module's want on node waits for an acting part's new version (§266): the first edge not run.
|
||||
func (m Module) actingWaiting(node string, n Nodes) (string, bool) {
|
||||
if m.Acting == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, e := range m.Acting(node) {
|
||||
if !n.Runs(e) {
|
||||
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", m.Name, node, e.Step, e.Part, short(e.Fp), e.Node), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// WantParts is the capability a node-engine of G states (ADR 0614 §4, design 47 G §525): it reads a want's `parts` and `kept-on`.
|
||||
const WantParts = "want-parts"
|
||||
|
||||
// KeptBuild is what a want names of a build: the digest of each of its parts by `<module>:<artifact>`, and, as the builder's
|
||||
// `build.ended` says it, the build node that keeps a kept build; for a trunk build the released version the catalog's
|
||||
// `version-released` says (design 47 G §379, §526, A-96).
|
||||
type KeptBuild struct {
|
||||
Parts map[string]string
|
||||
KeptOn string
|
||||
Version string
|
||||
}
|
||||
|
||||
// withKept is w naming the kept build at its fingerprint, the parts by digest and the keeping node, only while the node's
|
||||
// engine states WantParts (design 47 G §525, §526); a want of a build that is not kept, or of a node that does not state it,
|
||||
// is w.
|
||||
func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want {
|
||||
if m.KeptBuilds == nil {
|
||||
return w
|
||||
}
|
||||
k, ok := m.KeptBuilds(w.Fp)
|
||||
if !ok || !slices.Contains(n.Reads[node], WantParts) {
|
||||
return w
|
||||
}
|
||||
w.Parts, w.KeptOn, w.Version = k.Parts, k.KeptOn, k.Version
|
||||
return w
|
||||
// ReadsVersion says whether node's node-engine states that it reads a want by version (the word want:version of the seat
|
||||
// node-engine, design 47 H §772, H-A-45); a node whose engine does not is written its want held, awaiting its hand placement
|
||||
// (H-A-73). Nil: every node does.
|
||||
ReadsVersion func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
||||
|
||||
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return ComposedDigest{}, false
|
||||
}
|
||||
return m.Composed(node, fp)
|
||||
}
|
||||
|
||||
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
||||
// Nodes is what the pass reads of the nodes: whether each is online (the watchman's presence), each module's have and the
|
||||
// node-engine seat's last `running`.
|
||||
type Nodes struct {
|
||||
Online map[string]bool
|
||||
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
|
||||
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy
|
||||
// while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider
|
||||
// before its consumers*). Nil: the seat's events are not heard, and the have says it.
|
||||
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309). Nil: the seat's events
|
||||
// are not heard.
|
||||
Running map[string]map[string]nodeengine.Running
|
||||
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
|
||||
Providers map[string]map[string][]string
|
||||
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
|
||||
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
|
||||
// Reads is, by node, the reader capabilities its node-engine states (nodeengine.NodeRecord.ManifestReads, ADR 0614 §4).
|
||||
Reads map[string][]string
|
||||
}
|
||||
|
||||
// healthy says whether a have runs fp healthy.
|
||||
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
|
||||
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
|
||||
}
|
||||
|
||||
// Failed says whether a have's health check failed once the node is done moving.
|
||||
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
|
||||
|
||||
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
|
||||
// name, ADR 0528 clause 3). Empty when none is online.
|
||||
func Canary(nodes []string, online map[string]bool) string {
|
||||
sorted := slices.Sorted(slices.Values(nodes))
|
||||
for _, n := range sorted {
|
||||
if online[n] {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
|
||||
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
|
||||
// down and one module never blocks another.
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
|
||||
// without which there is nothing to want: the build done, not held, the grant and the composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
if m.BuildFailed {
|
||||
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError)
|
||||
}
|
||||
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP))
|
||||
case m.Held:
|
||||
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||
case m.MachineHeld != nil && m.MachineHeld(node):
|
||||
return false, mwc.RuleNotHeld, fmt.Sprintf("the condition machine.%s.network is open: no want moves to %s until its own network is healthy", node, node)
|
||||
}
|
||||
if ordering {
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
|
||||
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
}
|
||||
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
||||
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if words, lacking := m.readerLacking(node, n); lacking {
|
||||
return false, mwc.RuleComposed, words
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.actingWaiting(node, n); waiting {
|
||||
return false, mwc.RuleComposed, words
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, mwc.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.verbWaiting(node); waiting {
|
||||
return false, mwc.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.busStateWaiting(node); waiting {
|
||||
return false, mwc.RuleProvisions, words
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
|
||||
// manifest needs that the engine does not state, read as the baseline when it states none, a bootstrapped seat counting as
|
||||
// read by every engine (graph.MissingReads, ADR 0636 §2). It is held under rule 7 (the
|
||||
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
|
||||
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
|
||||
//
|
||||
// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a
|
||||
// first form, so a build written in one is never held by its forms; the words name the forms it reads none of.
|
||||
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
|
||||
missing := graph.MissingReads(m.Needs, n.Reads[node])
|
||||
if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) {
|
||||
missing = slices.Clone(m.FormNeeds)
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
|
||||
}
|
||||
|
||||
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
||||
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
||||
func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
if m.Provisions == nil {
|
||||
return "", false
|
||||
}
|
||||
if missing := m.MissingProvisions(node); len(missing) > 0 {
|
||||
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
||||
}
|
||||
keyed := m.Provisions(node)
|
||||
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
|
||||
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
|
||||
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
|
||||
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
|
||||
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
|
||||
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
|
||||
// bound (ADR 0603 §8).
|
||||
needed := m.Manifest.Requires
|
||||
if m.Answering != nil {
|
||||
needed = catalogue.Awaited(m.Manifest, m.Answering)
|
||||
have := map[string]bool{}
|
||||
for _, p := range keyed {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
|
||||
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
|
||||
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
|
||||
continue
|
||||
}
|
||||
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
|
||||
continue
|
||||
}
|
||||
needed = append(needed, r)
|
||||
if !have[r] {
|
||||
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
||||
if p := keyed[key]; p.State != meshprovisionerclient.Provided && slices.Contains(needed, p.Provision) {
|
||||
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
||||
if p.Why != "" {
|
||||
words += ": " + p.Why
|
||||
}
|
||||
return words, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
|
||||
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
|
||||
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
|
||||
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
|
||||
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
|
||||
func (m Module) MissingProvisions(node string) []string {
|
||||
if m.Provisions == nil || m.Answering == nil {
|
||||
return nil
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, p := range m.Provisions(node) {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
var out []string
|
||||
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
|
||||
if !have[r] {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
|
||||
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
|
||||
func (n Nodes) runsHealthy(node, module string) bool {
|
||||
@@ -363,312 +83,53 @@ func (n Nodes) runsHealthy(node, module string) bool {
|
||||
}
|
||||
|
||||
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
|
||||
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
|
||||
// module of which nothing was heard counts as not healthy. The have is never read for it.
|
||||
// (design 47 G §520). The have is never read for it.
|
||||
func (n Nodes) RunsHealthy(node, module, fp string) bool {
|
||||
r, ok := n.Running[node][module]
|
||||
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
|
||||
}
|
||||
|
||||
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
||||
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
||||
// offers it) and holds.
|
||||
func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
||||
providers := n.Providers[provision]
|
||||
if len(providers) == 0 {
|
||||
return true, ""
|
||||
}
|
||||
names := make([]string, 0, len(providers))
|
||||
for p := range providers {
|
||||
names = append(names, p)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, p := range names {
|
||||
nodes := providers[p]
|
||||
at := node
|
||||
if !slices.Contains(nodes, node) {
|
||||
at = Canary(nodes, n.Online)
|
||||
}
|
||||
if at != "" && n.runsHealthy(at, p) {
|
||||
return true, ""
|
||||
}
|
||||
}
|
||||
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
||||
}
|
||||
|
||||
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
|
||||
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
|
||||
func unreached(m Module, node string, n Nodes) string {
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return why
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
||||
// valuable or rebuildable; cache and none do not).
|
||||
func keepsData(m catalogue.Manifest) bool {
|
||||
for _, it := range m.DataItems() {
|
||||
if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
|
||||
// Outcome is what one module's pass decides: each node's want to write (only those that change).
|
||||
type Outcome struct {
|
||||
Wants map[string]mwc.Want
|
||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||
Held *mwc.Held
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
// back: the operator is asked whether to restore each (ADR 0539 §19).
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
|
||||
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
|
||||
Unreached map[string]string
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
// own composition and is never here.
|
||||
Neither map[string]Holding
|
||||
}
|
||||
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||
// controller's judgement of the move and its words.
|
||||
type RecordNeed struct {
|
||||
Node string
|
||||
Digest string
|
||||
Judgement Judgement
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
|
||||
// §2): no bar of its own.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words.
|
||||
func (h Holding) Reason() string {
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
// module holding data is not set back but asked about (Restore). A
|
||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome {
|
||||
out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
||||
held := m.Held
|
||||
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
||||
out.Lifted, held = true, false
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.Fp != m.FP || !failed(h) {
|
||||
continue
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
mm.Held = held
|
||||
// Wants is the pass of one module m: for each of its nodes, the want naming its Target, starting from the want written before
|
||||
// (wants, by node). A put that owns a node's want writes the put's; a node with no target keeps the want written. A node that
|
||||
// wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, wants map[string]mwc.Want) Outcome {
|
||||
out := Outcome{Wants: map[string]mwc.Want{}}
|
||||
for _, node := range m.Nodes {
|
||||
if m.FP == "" {
|
||||
break
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next mwc.Want
|
||||
p, put := m.Puts[node]
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||
// without hold is not kept this way: its put takes it further once heard.
|
||||
p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||
Step: puts.Holding}, true
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put
|
||||
// heard released gives it back to the rule.
|
||||
p, put = puts.Put{FP: before.Fp, Version: before.Version, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
if want != nil {
|
||||
w := *want
|
||||
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp {
|
||||
w.Composed = ul.Composed
|
||||
}
|
||||
if !had || !mwc.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
if want != nil && (!had || !mwc.SameWant(normal(before), normal(*want))) {
|
||||
out.Wants[node] = *want
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
if w, ok := m.userListRevision(node, before, had, n); ok {
|
||||
if !had || !mwc.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
t, ok := Target{}, false
|
||||
if m.Target != nil {
|
||||
t, ok = m.Target(node)
|
||||
}
|
||||
if !ok || t.Fp == "" {
|
||||
continue
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary {
|
||||
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
|
||||
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
|
||||
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
|
||||
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
|
||||
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
|
||||
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
|
||||
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
|
||||
if older {
|
||||
_, composed := m.composition(node, have.Fp)
|
||||
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
|
||||
}
|
||||
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
|
||||
switch {
|
||||
case older:
|
||||
ok, target = true, have.Fp
|
||||
case oldForm:
|
||||
// held back by rule 3 as written: on neither list until the canary runs F healthy
|
||||
default:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
}
|
||||
next := mwc.Want{Fp: t.Fp, Version: t.Version, Pipeline: t.Pipeline, Rank: t.Rank, Self: m.selfOn(node)}
|
||||
if had && !before.Removed && !before.Put && before.Fp == t.Fp {
|
||||
// The answers the want carries stay while its build does: a failed backup's (ADR 0539 §16), a held revert's, the yes.
|
||||
next.Backup, next.Words, next.Revert, next.Yes, next.Release = before.Backup, before.Words, before.Revert, before.Yes, before.Release
|
||||
}
|
||||
if why := unreached(m, node, n); why != "" {
|
||||
if out.Unreached == nil {
|
||||
out.Unreached = map[string]string{}
|
||||
}
|
||||
out.Unreached[node] = why
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) &&
|
||||
n.Haves[node][m.Name].Fp != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
d, _ := m.composition(node, target)
|
||||
next = n.withKept(m, node, mwc.Want{Fp: target, Composed: d.Digest})
|
||||
at := m
|
||||
if target != m.FP {
|
||||
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
||||
next.Newest, next.Rule, next.Words = m.FP, int64(rule), words
|
||||
at.FP = target
|
||||
}
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
||||
asked, hadLive := before, had
|
||||
if before.Removed {
|
||||
asked, hadLive = mwc.Want{}, false
|
||||
}
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
continue
|
||||
}
|
||||
case held && had && before.Fp == m.FP:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
last := lastHealthy[node]
|
||||
switch {
|
||||
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
|
||||
out.Restore = append(out.Restore, node)
|
||||
case last == "":
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
||||
default:
|
||||
d, there := m.composition(node, last)
|
||||
if !there {
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
|
||||
break
|
||||
}
|
||||
next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words}
|
||||
}
|
||||
default:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
if !had || before.Removed {
|
||||
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
h, there := n.Haves[node][m.Name]
|
||||
switch {
|
||||
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
|
||||
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||
next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||
if !mwc.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words}
|
||||
if before.Fp == m.FP {
|
||||
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
||||
next = mwc.Want{Fp: m.FP, Composed: before.Composed}
|
||||
} else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed {
|
||||
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
||||
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
||||
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
||||
at := m
|
||||
at.FP = before.Fp
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
if !wait {
|
||||
next.Composed = d.Digest
|
||||
}
|
||||
}
|
||||
}
|
||||
next, write := m.selfDerived(node, before, had, next, n, &out)
|
||||
if !write {
|
||||
continue
|
||||
if m.ReadsVersion != nil && !m.ReadsVersion(node) {
|
||||
// A node-engine that cannot read a want by version is placed by hand when its machine is back (design 47 H §772).
|
||||
next.Hold, next.Words, next.Backup, next.Revert = true, mwc.HoldAwaitsHandPlacement, "", ""
|
||||
}
|
||||
if !had || !mwc.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
@@ -682,143 +143,17 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
|
||||
return out
|
||||
}
|
||||
|
||||
// selfDerived is next with its Self derived from the assignments now (Module.Self), whichever rule made it, and whether it is
|
||||
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
||||
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
||||
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
||||
// assignments seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
||||
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
||||
// while its ask waits, the want written stays and the node's holding says why.
|
||||
func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) {
|
||||
switch {
|
||||
case next.Removed:
|
||||
return next, true
|
||||
case m.selfOn(node):
|
||||
next.Self, next.Composed = true, ""
|
||||
return next, true
|
||||
case !next.Self && next.Composed != "":
|
||||
return next, true
|
||||
}
|
||||
next.Self, next.Yes = false, ""
|
||||
d, there := m.composition(node, next.Fp)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
at.FP = next.Fp
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
|
||||
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
return next, false
|
||||
}
|
||||
next.Composed = d.Digest
|
||||
return next, true
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
||||
// the world's lastHealthy) already asks for phase running.
|
||||
// moving).
|
||||
func failed(h nodeengine.HaveByNodeModuleValue) bool {
|
||||
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h nodeengine.HaveByNodeModuleValue) bool {
|
||||
return h.Phase == nodeengineclient.PhaseRunning
|
||||
}
|
||||
|
||||
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
||||
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
||||
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
||||
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
||||
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
||||
func allRunHealthy(m Module, n Nodes) bool {
|
||||
ran := 0
|
||||
for _, node := range m.Nodes {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
|
||||
continue
|
||||
}
|
||||
if !n.RunsHealthy(node, m.Name, m.FP) {
|
||||
return false
|
||||
}
|
||||
ran++
|
||||
}
|
||||
return ran > 0
|
||||
}
|
||||
|
||||
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
||||
// it, and the node's own words. The node is the held record's own field.
|
||||
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
|
||||
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
||||
if !at.IsZero() {
|
||||
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if d := strings.TrimSpace(h.Detail); d != "" {
|
||||
return words + ": " + d
|
||||
}
|
||||
return words + ": the node gave no words"
|
||||
}
|
||||
|
||||
// LiftedWords say why a hold was lifted by itself.
|
||||
func LiftedWords(module, fp string) string {
|
||||
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
||||
}
|
||||
|
||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||
func normal(w mwc.Want) mwc.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
func short(fp string) string {
|
||||
part, err := catalogkey.KeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
@@ -827,69 +162,20 @@ func short(fp string) string {
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
|
||||
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
|
||||
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
|
||||
// streams before it applies (§2); the assigner asks no snapshot for it.
|
||||
func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
|
||||
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
|
||||
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
|
||||
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if m.Allowed != nil && m.Allowed(node, d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the digest the node-engine's have
|
||||
// waits on, its judgement of the move and its words (ADR 0539 §14).
|
||||
type RecordNeed struct {
|
||||
Node string
|
||||
Digest string
|
||||
Judgement Judgement
|
||||
Words string
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
|
||||
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
||||
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d ComposedDigest) bool {
|
||||
if m.Chosen != nil {
|
||||
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
if d.Policy != "" {
|
||||
return d.Policy == catalogue.PolicyRecord
|
||||
}
|
||||
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
|
||||
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
|
||||
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
|
||||
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
|
||||
func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) {
|
||||
if !catalogue.ProvidesBus(m.Manifest) {
|
||||
return mwc.Want{}, false
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) {
|
||||
return mwc.Want{}, false
|
||||
}
|
||||
d, ok := m.composition(node, running.Fp)
|
||||
if !ok || !d.UserListOnly {
|
||||
return mwc.Want{}, false
|
||||
}
|
||||
w := mwc.Want{Fp: running.Fp, Composed: d.Digest}
|
||||
if had && before.Composed == d.Digest {
|
||||
w.Backup, w.Words = before.Backup, before.Words
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g grantsstate.Grant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
// KeptBuild is what the assigner knows of a build: the digest of each of its parts by `<module>:<artifact>` and the build node that
|
||||
// keeps it, as the builder's `build.ended` says them, and its released version as the catalog's `version-released` says it (design
|
||||
// 47 G §379, A-96). A want names the version alone (design 47 H §750).
|
||||
type KeptBuild struct {
|
||||
Parts map[string]string
|
||||
KeptOn string
|
||||
Version string
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -53,9 +53,6 @@ type Computed struct {
|
||||
// Uncomputed is each placed module whose entry could not be computed, as "<node>/<module>: why", sorted: it is said,
|
||||
// never left out in silence.
|
||||
Uncomputed []string
|
||||
// Cycles are the cycles among the acting waits, within one step or across steps (Cycles, design 47 G §272, A-323): said, never
|
||||
// left in silence.
|
||||
Cycles []string
|
||||
}
|
||||
|
||||
// placedAt is one placed module with what its entry is computed from.
|
||||
@@ -74,7 +71,7 @@ type placedAt struct {
|
||||
optional func(string) bool
|
||||
}
|
||||
|
||||
type actingRule struct{ seat, step, on, bound string }
|
||||
type actingRule struct{ seat, step, on string }
|
||||
|
||||
// Compute is the assignment graph revision of in: one entry per module placed on a node whose branch's catalog graph
|
||||
// holds it, its version the build the placement names or else the catalog entry's fingerprint (design 47 lines 909-914),
|
||||
@@ -128,7 +125,7 @@ func Compute(in Inputs) Computed {
|
||||
p.langs[lang] = true
|
||||
}
|
||||
for _, r := range ce.Acts {
|
||||
p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On, bound: r.Bound})
|
||||
p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On})
|
||||
}
|
||||
p.provides, p.contributes = ce.Provides, ce.Contributes
|
||||
p.delivering = map[string]seatdef.Definition{}
|
||||
@@ -199,7 +196,7 @@ func Compute(in Inputs) Computed {
|
||||
}
|
||||
for _, h := range from {
|
||||
t.entry.Edges = append(t.entry.Edges, meshwants.Edge{Kind: EdgeActing, Node: h.node, Module: part, Fp: h.entry.Fp,
|
||||
Seat: rule.seat, Step: rule.step, Bound: rule.bound})
|
||||
Seat: rule.seat, Step: rule.step})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -217,7 +214,6 @@ func Compute(in Inputs) Computed {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: p.node, Module: p.placed.Module, Digest: d})
|
||||
}
|
||||
out.Revision = rev
|
||||
out.Cycles = Cycles(slices.Collect(maps.Values(out.Entries)))
|
||||
slices.Sort(out.Uncomputed)
|
||||
return out
|
||||
}
|
||||
@@ -229,7 +225,7 @@ func sortEdges(edges []meshwants.Edge) {
|
||||
}
|
||||
|
||||
func edgeKey(e meshwants.Edge) string {
|
||||
return strings.Join([]string{e.Kind, e.Node, e.Module, e.Provision, e.Seat, e.Step, e.Fp, e.Bound}, "\x00")
|
||||
return strings.Join([]string{e.Kind, e.Node, e.Module, e.Provision, e.Seat, e.Step, e.Fp}, "\x00")
|
||||
}
|
||||
|
||||
// EntryDigest is the digest an entry is stored under: the same entry, the same digest.
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
package assigngraph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Cycles are the cycles among the acting waits (design 47 G §272, ADR 0656, A-323): an entry waits for its acting parts in the step
|
||||
// an edge names, so acting parts that wait for each other, in one step or across steps, wait for ever unless a rule breaks the
|
||||
// cycle. A cycle within one step is `<step>: a -> b -> a`; one across steps names each edge's step, `a -[compose]-> b -[place]-> a`;
|
||||
// each by node and module, said once from its smallest member, sorted. A holder's own inputs are no edge here (the graph gives an
|
||||
// acting edge only from a part to the entries it acts on, never to what the part is built from).
|
||||
func Cycles(entries []meshwants.AssignmentEntry) []string {
|
||||
type key struct{ step, at string }
|
||||
next := map[key][]string{}
|
||||
steps := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, edge := range e.Edges {
|
||||
if edge.Kind != EdgeActing {
|
||||
continue
|
||||
}
|
||||
k := key{edge.Step, e.Node + "/" + e.Module}
|
||||
next[k] = append(next[k], edge.Node+"/"+edge.Module)
|
||||
steps[edge.Step] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, step := range slices.Sorted(maps.Keys(steps)) {
|
||||
const (
|
||||
open = iota + 1
|
||||
done
|
||||
)
|
||||
state := map[string]int{}
|
||||
var path []string
|
||||
var walk func(at string)
|
||||
walk = func(at string) {
|
||||
state[at] = open
|
||||
path = append(path, at)
|
||||
for _, to := range next[key{step, at}] {
|
||||
switch state[to] {
|
||||
case 0:
|
||||
walk(to)
|
||||
case open:
|
||||
from := slices.Index(path, to)
|
||||
cycle := slices.Clone(path[from:])
|
||||
// start at the smallest member so one cycle is said once, whichever member is met first
|
||||
lowest := slices.Index(cycle, slices.Min(cycle))
|
||||
cycle = append(cycle[lowest:], cycle[:lowest]...)
|
||||
out = append(out, fmt.Sprintf("%s: %s -> %s", step, strings.Join(cycle, " -> "), cycle[0]))
|
||||
}
|
||||
}
|
||||
path = path[:len(path)-1]
|
||||
state[at] = done
|
||||
}
|
||||
starts := map[string]bool{}
|
||||
for k := range next {
|
||||
if k.step == step {
|
||||
starts[k.at] = true
|
||||
}
|
||||
}
|
||||
for _, at := range slices.Sorted(maps.Keys(starts)) {
|
||||
if state[at] == 0 {
|
||||
walk(at)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Across steps (A-323): the shortest cycle through each entry over every acting edge, said when its edges name two steps or more.
|
||||
for _, hops := range cyclesThrough(entries, func(_ meshwants.AssignmentEntry, e meshwants.Edge) bool { return e.Kind == EdgeActing }) {
|
||||
if crossesSteps(hops) {
|
||||
out = append(out, hopWords(rotated(hops)))
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
// HeldCycles is, by `<node>/<module>`, the cycle each entry lies on among the edges holds says hold it (A-323, A-330): a want held
|
||||
// on such a cycle is held for ever, since no rule breaks it. Each cycle starts at the entry and names each edge's step.
|
||||
func HeldCycles(entries []meshwants.AssignmentEntry, holds func(e meshwants.AssignmentEntry, edge meshwants.Edge) bool) map[string]string {
|
||||
out := map[string]string{}
|
||||
for at, hops := range cyclesThrough(entries, holds) {
|
||||
out[at] = hopWords(hops)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Engines is, by node, the module that holds the seat node-engine there, read from the acting edges' seat (A-330).
|
||||
func Engines(entries []meshwants.AssignmentEntry) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, e := range entries {
|
||||
for _, edge := range e.Edges {
|
||||
if edge.Kind == EdgeActing && edge.Seat == nodeengine.Seat {
|
||||
out[edge.Node] = edge.Module
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Runner is the module that holds the seat mesh-pipeline-runner, read from the acting edges' seat as Engines reads the node-engine
|
||||
// (A-334); "" when none is read.
|
||||
func Runner(entries []meshwants.AssignmentEntry) string {
|
||||
for _, e := range entries {
|
||||
for _, edge := range e.Edges {
|
||||
if edge.Kind == EdgeActing && edge.Seat == meshpipelinerunner.Seat {
|
||||
return edge.Module
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hop is one edge of a cycle: from the entry at, in step, to the next hop's entry.
|
||||
type hop struct{ at, step string }
|
||||
|
||||
// cyclesThrough is, by `<node>/<module>`, the shortest cycle through each entry over the edges keep takes, found within the
|
||||
// entry's strongly connected part, each starting at the entry.
|
||||
func cyclesThrough(entries []meshwants.AssignmentEntry, keep func(meshwants.AssignmentEntry, meshwants.Edge) bool) map[string][]hop {
|
||||
type out struct{ to, step string }
|
||||
next := map[string][]out{}
|
||||
for _, e := range entries {
|
||||
at := e.Node + "/" + e.Module
|
||||
for _, edge := range e.Edges {
|
||||
if keep(e, edge) {
|
||||
next[at] = append(next[at], out{edge.Node + "/" + edge.Module, edge.Step})
|
||||
}
|
||||
}
|
||||
}
|
||||
for at := range next {
|
||||
slices.SortFunc(next[at], func(a, b out) int { return strings.Compare(a.to+"\x00"+a.step, b.to+"\x00"+b.step) })
|
||||
}
|
||||
part := strongParts(slices.Sorted(maps.Keys(next)), func(at string) []string {
|
||||
var to []string
|
||||
for _, o := range next[at] {
|
||||
to = append(to, o.to)
|
||||
}
|
||||
return to
|
||||
})
|
||||
found := map[string][]hop{}
|
||||
for _, start := range slices.Sorted(maps.Keys(next)) {
|
||||
// breadth first from start back to it, within its part
|
||||
prev := map[string]hop{}
|
||||
queue := []string{start}
|
||||
seen := map[string]bool{}
|
||||
for len(queue) > 0 && found[start] == nil {
|
||||
at := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, o := range next[at] {
|
||||
if part[o.to] != part[start] {
|
||||
continue
|
||||
}
|
||||
if o.to == start {
|
||||
hops := []hop{{at, o.step}}
|
||||
for cur := at; cur != start; {
|
||||
h := prev[cur]
|
||||
hops = append(hops, h)
|
||||
cur = h.at
|
||||
}
|
||||
slices.Reverse(hops)
|
||||
found[start] = hops
|
||||
break
|
||||
}
|
||||
if !seen[o.to] {
|
||||
seen[o.to] = true
|
||||
prev[o.to] = hop{at, o.step}
|
||||
queue = append(queue, o.to)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// strongParts numbers each vertex by its strongly connected part (Tarjan).
|
||||
func strongParts(vertices []string, succ func(string) []string) map[string]int {
|
||||
index, low, part := map[string]int{}, map[string]int{}, map[string]int{}
|
||||
on := map[string]bool{}
|
||||
var stack []string
|
||||
n, parts := 0, 0
|
||||
var visit func(v string)
|
||||
visit = func(v string) {
|
||||
index[v], low[v] = n, n
|
||||
n++
|
||||
stack = append(stack, v)
|
||||
on[v] = true
|
||||
for _, w := range succ(v) {
|
||||
if _, ok := index[w]; !ok {
|
||||
visit(w)
|
||||
low[v] = min(low[v], low[w])
|
||||
} else if on[w] {
|
||||
low[v] = min(low[v], index[w])
|
||||
}
|
||||
}
|
||||
if low[v] == index[v] {
|
||||
parts++
|
||||
for {
|
||||
w := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
on[w] = false
|
||||
part[w] = parts
|
||||
if w == v {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, v := range vertices {
|
||||
if _, ok := index[v]; !ok {
|
||||
visit(v)
|
||||
}
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
func crossesSteps(hops []hop) bool {
|
||||
for _, h := range hops[1:] {
|
||||
if h.step != hops[0].step {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rotated is the cycle started at its smallest member, so one cycle is said once.
|
||||
func rotated(hops []hop) []hop {
|
||||
lowest := 0
|
||||
for i, h := range hops {
|
||||
if h.at < hops[lowest].at {
|
||||
lowest = i
|
||||
}
|
||||
}
|
||||
return append(slices.Clone(hops[lowest:]), hops[:lowest]...)
|
||||
}
|
||||
|
||||
// hopWords is a cycle with each edge's step: `a -[compose]-> b -[place]-> a`.
|
||||
func hopWords(hops []hop) string {
|
||||
var b strings.Builder
|
||||
for _, h := range hops {
|
||||
fmt.Fprintf(&b, "%s -[%s]-> ", h.at, h.step)
|
||||
}
|
||||
return b.String() + hops[0].at
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package assigngraph
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G §272 (AS10), A-323: a cycle among the waits is found within one step and across steps, each edge's step named across
|
||||
// steps; a chain and provisioning edges are none.
|
||||
func TestACycleAmongTheWaitsWithinOrAcrossStepsIsFound(t *testing.T) {
|
||||
act := func(node, module, step string) meshwants.Edge {
|
||||
return meshwants.Edge{Kind: EdgeActing, Node: node, Module: module, Step: step}
|
||||
}
|
||||
entry := func(module string, edges ...meshwants.Edge) meshwants.AssignmentEntry {
|
||||
return meshwants.AssignmentEntry{Node: "ace", Module: module, Edges: edges}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
entries []meshwants.AssignmentEntry
|
||||
want []string
|
||||
}{
|
||||
{"two parts on each other in one step", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "a", "compose"))},
|
||||
[]string{"compose: ace/a -> ace/b -> ace/a"}},
|
||||
{"three in a ring, said once", []meshwants.AssignmentEntry{entry("c", act("ace", "a", "build")), entry("a", act("ace", "b", "build")), entry("b", act("ace", "c", "build"))},
|
||||
[]string{"build: ace/a -> ace/b -> ace/c -> ace/a"}},
|
||||
{"the indexer taking in the checker, the checker checking the indexer", []meshwants.AssignmentEntry{entry("checker", act("ace", "indexer", "take-in")), entry("indexer", act("ace", "checker", "check"))},
|
||||
[]string{"ace/checker -[take-in]-> ace/indexer -[check]-> ace/checker"}},
|
||||
{"the assigner and the node-engine, across compose and place", []meshwants.AssignmentEntry{entry("assigner", act("ace", "engine", "compose")),
|
||||
entry("engine", act("ace", "assigner", "place")), entry("app", act("ace", "engine", "compose"), act("ace", "assigner", "place"))},
|
||||
[]string{"ace/assigner -[compose]-> ace/engine -[place]-> ace/assigner"}},
|
||||
{"a chain", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "c", "compose")), entry("c")}, nil},
|
||||
{"provisioning edges are no wait of a step", []meshwants.AssignmentEntry{entry("a", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "b"}), entry("b", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "a"})}, nil},
|
||||
} {
|
||||
if got := Cycles(c.entries); !slices.Equal(got, c.want) {
|
||||
t.Errorf("%s: cycles %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A-323, A-330: the node-engine on a node is read from the acting edges' seat, and the cycles that hold a want are those of the
|
||||
// edges said to hold it, each from the entry, naming each edge's step.
|
||||
func TestTheHeldCyclesAreThoseOfTheHoldingEdges(t *testing.T) {
|
||||
entries := []meshwants.AssignmentEntry{
|
||||
{Node: "ace", Module: "assigner", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "engine", Step: "compose", Seat: "node-engine"}}},
|
||||
{Node: "ace", Module: "engine", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "assigner", Step: "place", Seat: "mesh-wants"}}},
|
||||
}
|
||||
if got := Engines(entries); len(got) != 1 || got["ace"] != "engine" {
|
||||
t.Errorf("the node-engines %v, want engine on ace", got)
|
||||
}
|
||||
all := func(meshwants.AssignmentEntry, meshwants.Edge) bool { return true }
|
||||
if got := HeldCycles(entries, all); len(got) != 2 || got["ace/engine"] != "ace/engine -[place]-> ace/assigner -[compose]-> ace/engine" {
|
||||
t.Errorf("held cycles %v", got)
|
||||
}
|
||||
// the node-engine's want held by no acting edge: no cycle holds anything
|
||||
if got := HeldCycles(entries, func(e meshwants.AssignmentEntry, _ meshwants.Edge) bool { return e.Module != "engine" }); len(got) != 0 {
|
||||
t.Errorf("held cycles with the node-engine's edges holding nothing: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) {
|
||||
}
|
||||
want := []meshwants.Edge{
|
||||
{Kind: EdgeActing, Node: "anchor", Module: "build-go", Fp: "src2:b1", Seat: "node-build-go", Step: "build"},
|
||||
{Kind: EdgeActing, Node: "anchor", Module: "node-engine", Fp: "src2:e1", Seat: "node-engine", Step: "compose", Bound: "30m"},
|
||||
{Kind: EdgeActing, Node: "anchor", Module: "node-engine", Fp: "src2:e1", Seat: "node-engine", Step: "compose"}, // the edge stays; the wait's bound goes (H-A-70)
|
||||
{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"},
|
||||
}
|
||||
if fmt.Sprint(webA.Edges) != fmt.Sprint(want) {
|
||||
|
||||
@@ -30,6 +30,29 @@ type Objects interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// Described is one object as the bucket's listing says it: its name, and for an entry object the module and fingerprint put beside
|
||||
// its bytes (empty for one put before them).
|
||||
type Described struct {
|
||||
Name, Module, Fp string
|
||||
}
|
||||
|
||||
// Describer is an object bucket whose listing says, in one read, each entry object's module and fingerprint, so the removal of the
|
||||
// revisions not current reads no object per entry (design 47 H-A-259).
|
||||
type Describer interface {
|
||||
// PutDescribed stores data whole under name, with module and fp beside it in the listing.
|
||||
PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error
|
||||
// Described is every object, as the listing says it.
|
||||
Described(ctx context.Context) ([]Described, error)
|
||||
}
|
||||
|
||||
// PutEntry stores the entry object e under name, with its module and fingerprint beside it where the bucket says them.
|
||||
func PutEntry(ctx context.Context, o Objects, name string, data []byte, e meshwants.AssignmentEntry) error {
|
||||
if d, ok := o.(Describer); ok {
|
||||
return d.PutDescribed(ctx, name, data, e.Module, e.Fp)
|
||||
}
|
||||
return o.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
// EntryObject is the object name of the entry of that digest (the seat's key form entry.<digest>).
|
||||
func EntryObject(digest string) string { return "entry." + digest }
|
||||
|
||||
@@ -144,7 +167,7 @@ func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.Assi
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := o.Put(ctx, name, data); err != nil {
|
||||
if err := PutEntry(ctx, o, name, data, e); err != nil {
|
||||
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
|
||||
}
|
||||
held[name] = true
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
client "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -356,5 +357,30 @@ func TestAssignments(t *testing.T) {
|
||||
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); h != nil {
|
||||
t.Errorf("a node seat nobody holds there is held by %v", h)
|
||||
}
|
||||
|
||||
// A-400: a seat no holding names is held by the first assigned module whose manifest claims it; a holding on record wins.
|
||||
assigned := map[string]seat.Assignments{
|
||||
"ace": {Node: "ace", Modules: []seat.PlacedModule{{Module: "mesh-delivery-assigner"}, {Module: "mesh-files"}}},
|
||||
"novox": {Node: "novox", Modules: []seat.PlacedModule{{Module: "nats"}, {Module: "mesh-files"}}, Holds: []seat.Holding{{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}},
|
||||
}
|
||||
manifests := map[string]catalogue.Manifest{
|
||||
"mesh-delivery-assigner": {Claims: []catalogue.Claim{{Name: "mesh-module-assigner", Scope: "mesh"}}},
|
||||
"nats": {Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}},
|
||||
"mesh-files": {Claims: []catalogue.Claim{{Name: "node-files"}}},
|
||||
}
|
||||
with, derived := SeatHoldersWithClaimants(assigned, manifests)
|
||||
keyed = map[string][]byte{}
|
||||
for k, v := range with {
|
||||
keyed[k], _ = json.Marshal(v)
|
||||
}
|
||||
if h, err := client.HoldersOf(keyed, "mesh-module-assigner", ""); err != nil || len(h) != 1 || h[0].Module != "mesh-delivery-assigner" || h[0].Node != "ace" {
|
||||
t.Errorf("an unheld seat's claimant is %v, %v", h, err)
|
||||
}
|
||||
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); len(h) != 1 || h[0].Module != "mesh-files" {
|
||||
t.Errorf("a node seat's claimant on novox is %v", h)
|
||||
}
|
||||
if h, _ := client.HoldersOf(keyed, "mesh-broker", ""); len(h) != 1 || h[0].Node != "novox" || len(derived) != 3 {
|
||||
t.Errorf("a held seat is %v; derived %v", h, derived)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ func DecodeCandidate(data []byte) (seat.CandidatesByProposalNodeValue, error) {
|
||||
if err := keys.CheckVersion(data); err != nil {
|
||||
return seat.CandidatesByProposalNodeValue{}, err
|
||||
}
|
||||
c, err := seat.DecodeCandidatesByProposalNodeValue(data)
|
||||
c, _, err := seat.DecodeCandidatesByProposalNodeValue(data)
|
||||
if err != nil {
|
||||
return seat.CandidatesByProposalNodeValue{}, fmt.Errorf("assignments: a candidate does not fit its shape: %w", err)
|
||||
}
|
||||
|
||||
@@ -252,3 +252,54 @@ func sortedKeys[V any](m map[string]V) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// SeatHoldersWithClaimants is SeatHoldersOf over the assignments with, for each seat no holding names (a seat held once:
|
||||
// on no node; a seat of every node: on that node), the first module assigned to a node (nodes and modules in order) whose
|
||||
// manifest claims it, as its holder (lead answer A-400, the same rule as grants' A-398; the operator, 2026-11-14: who holds
|
||||
// a seat follows from the module graph and the assignments). derived lists each key so held, for its writer to say. Pure.
|
||||
func SeatHoldersWithClaimants(assignments map[string]seat.Assignments, manifests map[string]catalogue.Manifest) (out map[string]seat.SeatHolders, derived []string) {
|
||||
held := map[string]bool{} // "<seat>" for a seat held once, "<seat>.<node>" for a seat of every node
|
||||
keyOf := func(claim, scope, node string) string {
|
||||
name := claim
|
||||
if s, ok := catalogue.SeatNamed(claim); ok {
|
||||
name = s.Name
|
||||
}
|
||||
if scope == catalogue.ScopeNode {
|
||||
return name + "." + node
|
||||
}
|
||||
return name
|
||||
}
|
||||
for _, n := range sortedKeys(assignments) {
|
||||
for _, h := range assignments[n].Holds {
|
||||
held[keyOf(h.Claim, h.Scope, n)] = true
|
||||
}
|
||||
}
|
||||
with := map[string]seat.Assignments{}
|
||||
var added []string
|
||||
for _, n := range sortedKeys(assignments) {
|
||||
a := assignments[n]
|
||||
a.Holds = append([]seat.Holding(nil), a.Holds...)
|
||||
modules := make([]string, 0, len(a.Modules))
|
||||
for _, m := range a.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
k := keyOf(c.Name, c.At(), n)
|
||||
if held[k] {
|
||||
continue
|
||||
}
|
||||
held[k] = true
|
||||
a.Holds = append(a.Holds, seat.Holding{Claim: c.Name, Module: module, Scope: c.At()})
|
||||
added = append(added, k+" by "+module+" on "+n)
|
||||
}
|
||||
}
|
||||
with[n] = a
|
||||
}
|
||||
return SeatHoldersOf(with), added
|
||||
}
|
||||
|
||||
@@ -428,7 +428,7 @@ func DecodeProposal(data []byte) (seat.Proposal, error) {
|
||||
if err := keys.CheckVersion(data); err != nil {
|
||||
return seat.Proposal{}, err
|
||||
}
|
||||
p, err := seat.DecodeProposalsByProposalValue(data)
|
||||
p, _, err := seat.DecodeProposalsByProposalValue(data)
|
||||
if err != nil {
|
||||
return seat.Proposal{}, fmt.Errorf("assignments: a proposal does not fit its shape: %w", err)
|
||||
}
|
||||
|
||||
@@ -18,17 +18,14 @@ import (
|
||||
const (
|
||||
// WaitsState holds the waits on an outside cause, one key `<node>.<module>.<fingerprint hex>` (design 47 G §706).
|
||||
WaitsState = "waits"
|
||||
// DeliveredAgainState holds each ask of assigner_delivered_again, one key `<module>`, its value the version, who asked and why, and
|
||||
// why it does not hold yet (design 47 G §376, A-250).
|
||||
DeliveredAgainState = "delivered-again"
|
||||
// DeliveryWaitsState holds each dependent's delivery wait the assigner said, one key `<module>`, its value the wait-held said
|
||||
// (design 47 G §376, §498, A-246).
|
||||
DeliveryWaitsState = "delivery-waits"
|
||||
// PutWaitsState holds each put wait the assigner said, one key `<node>.<module>`, its value the wait's id and the put as said
|
||||
// (design 47 G A-204, A-211), so a put that ends while the assigner is stopped gets its wait-ended at the next start.
|
||||
PutWaitsState = "put-waits"
|
||||
// InputsState holds the inputs of the composition steps, `<time>.<digest>`, which the seat mesh-wants' verb input answers (design 47 G §354).
|
||||
InputsState = "inputs"
|
||||
// CompositionRunsState holds the composition runs, one key `<candidate key>.<machine>` per run and its bound keys
|
||||
// `bound.<run key>` and `untaken.<run key>` (design 47 G §719, §720, A-470).
|
||||
CompositionRunsState = "composition-runs"
|
||||
// PoliciesState is the state of the policies a person set for a module's upgrade.
|
||||
PoliciesState = "policies"
|
||||
)
|
||||
@@ -37,16 +34,19 @@ const (
|
||||
// constant of its name (design 47 G A-296, §428).
|
||||
func own(state string) string { return plc.Module() + "_" + state }
|
||||
|
||||
func Wants() string { return own(wants.StateWants) }
|
||||
func Differences() string { return own(wants.StateDifferences) }
|
||||
func Asks() string { return own("asks") }
|
||||
func Puts() string { return own("puts") }
|
||||
func Waits() string { return own(WaitsState) }
|
||||
func DeliveredAgain() string { return own(DeliveredAgainState) }
|
||||
func DeliveryWaits() string { return own(DeliveryWaitsState) }
|
||||
func PutWaits() string { return own(PutWaitsState) }
|
||||
func Inputs() string { return own(InputsState) }
|
||||
func Policies() string { return own(PoliciesState) }
|
||||
func Wants() string { return own(wants.StateWants) }
|
||||
func Differences() string { return own(wants.StateDifferences) }
|
||||
func Asks() string { return own("asks") }
|
||||
func Puts() string { return own("puts") }
|
||||
func Waits() string { return own(WaitsState) }
|
||||
func PutWaits() string { return own(PutWaitsState) }
|
||||
func Inputs() string { return own(InputsState) }
|
||||
func CompositionRuns() string { return own(CompositionRunsState) }
|
||||
func Policies() string { return own(PoliciesState) }
|
||||
|
||||
// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their
|
||||
// drops; named as mesh-sdk's process.TakenState.
|
||||
func Taken() string { return own("taken") }
|
||||
|
||||
// AsksHistory is the revisions the bucket of asks keeps per key.
|
||||
const AsksHistory = 5
|
||||
@@ -67,10 +67,6 @@ const (
|
||||
Graph = IndexerModule + "_graph"
|
||||
ModulesRead = IndexerModule + "." + catalog.StateModules
|
||||
ManifestsRead = IndexerModule + "." + catalog.StateManifests
|
||||
// ProvisionerModule is the holder of the seat mesh-provisioner, named from this module's manifest (reads its provisions); Provisions
|
||||
// is its state provisions.
|
||||
ProvisionerModule = "mesh-provisioner"
|
||||
Provisions = ProvisionerModule + "_provisions"
|
||||
// TellerModule is the module whose hand the operator's answers pass: the one that holds each ask's answer, and the
|
||||
// name the hand-act log records a warrant under. Held equal to the manifest's reads by its test.
|
||||
TellerModule = "mesh-delivery-teller"
|
||||
|
||||
@@ -12,9 +12,9 @@ func TestBucketsAreTheEarlierNames(t *testing.T) {
|
||||
for name, c := range map[string][2]string{
|
||||
"Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"},
|
||||
"Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"},
|
||||
"Answers": {Answers, "mesh-delivery-teller_answers"},
|
||||
"Grants": {Grants, "mesh-grants_grants"},
|
||||
"Provisions": {Provisions, "mesh-provisioner_provisions"}, "TellerModule": {TellerModule, "mesh-delivery-teller"},
|
||||
"Answers": {Answers, "mesh-delivery-teller_answers"},
|
||||
"Grants": {Grants, "mesh-grants_grants"},
|
||||
"TellerModule": {TellerModule, "mesh-delivery-teller"},
|
||||
} {
|
||||
if c[0] != c[1] {
|
||||
t.Errorf("%s = %q, want %q", name, c[0], c[1])
|
||||
@@ -31,7 +31,8 @@ func TestTheAssignersOwnBucketsTakeTheModuleItRunsAs(t *testing.T) {
|
||||
"Wants": {Wants(), "some-assigner_wants"}, "Differences": {Differences(), "some-assigner_differences"},
|
||||
"Asks": {Asks(), "some-assigner_asks"}, "Puts": {Puts(), "some-assigner_puts"}, "Policies": {Policies(), "some-assigner_policies"},
|
||||
"Waits": {Waits(), "some-assigner_waits"}, "Inputs": {Inputs(), "some-assigner_inputs"},
|
||||
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
|
||||
"CompositionRuns": {CompositionRuns(), "some-assigner_composition-runs"},
|
||||
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
|
||||
} {
|
||||
if c[0] != c[1] {
|
||||
t.Errorf("%s = %q, want %q", name, c[0], c[1])
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user