Compare commits
584
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
85d1707e82 | ||
|
|
cb6cd9d996 | ||
|
|
2057c9d304 | ||
|
|
6ac8952541 | ||
|
|
6e89a5defd | ||
|
|
4b11f66b4b | ||
|
|
0439af3649 | ||
|
|
5cc26e2659 | ||
|
|
91dc064f92 | ||
|
|
7b6468e2bb | ||
|
|
0cc1728a04 | ||
|
|
d1768241dc | ||
|
|
cdd8bd3cba | ||
|
|
121a8090b3 | ||
|
|
ba9d4787f9 | ||
|
|
87cfae48b2 | ||
|
|
179a4a6402 | ||
|
|
7bfeb466bf | ||
|
|
e7803f8943 | ||
|
|
baeb87d195 | ||
|
|
9172eccbc4 | ||
|
|
07ea73bd64 | ||
|
|
d57e2325f6 | ||
|
|
43c812e98e | ||
|
|
e3d994fde1 | ||
|
|
eb89cb7da3 | ||
|
|
ed7cefb5d8 | ||
|
|
f21b8c202d | ||
|
|
b4bf772de4 | ||
|
|
c5f76cb974 | ||
|
|
8700aabba3 | ||
|
|
06781d4dbc | ||
|
|
d0b208b668 | ||
|
|
85ad1eef74 | ||
|
|
51b89964c5 | ||
|
|
1a6a380507 | ||
|
|
0612916af0 | ||
|
|
10a6824abe | ||
|
|
bb285f9c1b | ||
|
|
cbed32524a | ||
|
|
b132653fc0 | ||
|
|
523ad97e28 | ||
|
|
5b4484efd8 | ||
|
|
0a43489a74 | ||
|
|
dbb77efab1 | ||
|
|
1b9f9355b5 | ||
|
|
72385fd84a | ||
|
|
3bb895152f | ||
|
|
c6d893e141 | ||
|
|
310ff0cabc | ||
|
|
13fcfae051 | ||
|
|
f8f33e54a7 | ||
|
|
869d15e6d3 | ||
|
|
292aa54bbb | ||
|
|
f539cdb7ec | ||
|
|
3b365ba7cc | ||
|
|
6eb41a17e1 | ||
|
|
a1cd4a1a2f | ||
|
|
7a30c0d54c | ||
|
|
a9eb4372d9 | ||
|
|
c025bb5f1e | ||
|
|
5a3510b115 | ||
|
|
7f892a757c | ||
|
|
40ed0633e0 | ||
|
|
5bbde5db40 | ||
|
|
21e452df28 | ||
|
|
dc32afdd7e | ||
|
|
8a0c39280d | ||
|
|
a00cccfe0d | ||
|
|
592b4002bd | ||
|
|
0adc5ab4e6 | ||
|
|
6079f634d6 | ||
|
|
eaf011e971 | ||
|
|
1c3b2597ba | ||
|
|
fd023f7473 | ||
|
|
5caa760f5e | ||
|
|
d63aae1d55 | ||
|
|
f8acfaeed7 | ||
|
|
0a9d1efe89 | ||
|
|
96a8690639 | ||
|
|
7868c9f365 | ||
|
|
225f2abb1f | ||
|
|
61806a7a92 | ||
|
|
61c42f9fbc | ||
|
|
86048f0ed9 | ||
|
|
039415cdd9 | ||
|
|
fb492da7e7 | ||
|
|
a12417fc13 | ||
|
|
8656e57eb9 | ||
|
|
650910dc9e | ||
|
|
321a0744d8 | ||
|
|
860ccd7e30 | ||
|
|
23799a30ed | ||
|
|
e558510a54 | ||
|
|
1d409cd852 | ||
|
|
302dcc098f | ||
|
|
3167f0104f | ||
|
|
0193949efe | ||
|
|
8dc186944e | ||
|
|
905265a56b | ||
|
|
4ef07521eb | ||
|
|
a9b9709911 | ||
|
|
df1412081b | ||
|
|
8e993c759a | ||
|
|
0c37ac378c | ||
|
|
d18bf8b3fd | ||
|
|
bef467febe | ||
|
|
a1a328c43e | ||
|
|
77e6e48ecd | ||
|
|
8734d5ee1a | ||
|
|
a3fea9106a | ||
|
|
4b215ba6ae | ||
|
|
4021de1d42 | ||
|
|
9afb287b9d | ||
|
|
c2710f52dd | ||
|
|
a3b646b3c9 | ||
|
|
732def66b0 | ||
|
|
ac3422f914 | ||
|
|
4d656fe4e1 | ||
|
|
94f6c3dba4 | ||
|
|
7d06ef7997 | ||
|
|
efee30e243 | ||
|
|
e8bff57b55 | ||
|
|
c6df8fa20f | ||
|
|
054ca5a6b3 | ||
|
|
b3b13f2ae1 | ||
|
|
0f108fd8ac | ||
|
|
0dd32d9722 | ||
|
|
31697e696d | ||
|
|
6441ae7e96 | ||
|
|
5694f6c39b | ||
|
|
ece79788a3 | ||
|
|
dc1aee51d9 | ||
|
|
cf378f9a25 | ||
|
|
2b916ac043 | ||
|
|
7c7b5e2d68 | ||
|
|
5173f2a31c | ||
|
|
97d3df3e59 | ||
|
|
7703162c95 | ||
|
|
38e13ef030 | ||
|
|
189ed10c36 | ||
|
|
75f5c64646 | ||
|
|
5d3b446988 | ||
|
|
ce01ec1f68 | ||
|
|
602ed830e9 | ||
|
|
a073a001c5 | ||
|
|
45e2b946c7 | ||
|
|
b41a0d59b6 | ||
|
|
f3813a279e | ||
|
|
420087ab14 | ||
|
|
26581d87ed | ||
|
|
730860b274 | ||
|
|
decbce4c54 | ||
|
|
14c22b4a45 | ||
|
|
e64c8ad0ca | ||
|
|
6304332ad5 | ||
|
|
783a2d2207 | ||
|
|
d581acdeda | ||
|
|
9cd03a7430 | ||
|
|
5aa6d39a98 | ||
|
|
4685610c19 | ||
|
|
f50442ac7a | ||
|
|
5c353cfc7b | ||
|
|
a484d5f960 | ||
|
|
4af6c274b8 | ||
|
|
7bf7e29a32 | ||
|
|
81bc0cc388 | ||
|
|
9d8647aa0f | ||
|
|
aa649d85a0 | ||
|
|
bcd815a6bb | ||
|
|
6e461bd24a | ||
|
|
ef41fa2e27 | ||
|
|
868ee6beb0 | ||
|
|
7c483da93a | ||
|
|
17a4ac620c | ||
|
|
95f45b1a74 | ||
|
|
2019122379 | ||
|
|
e504c12940 | ||
|
|
25d677d20c | ||
|
|
6bde46cb00 | ||
|
|
3ea81943f5 | ||
|
|
3dc4307aec | ||
|
|
5f37a43b91 | ||
|
|
9b55bcb127 | ||
|
|
9f6801a6c5 | ||
|
|
22a37b758c | ||
|
|
25aa110920 | ||
|
|
71508cc962 | ||
|
|
87bdc95a3d | ||
|
|
7b42c94a4d | ||
|
|
6dd8e08a84 | ||
|
|
60f413f9bc | ||
|
|
5108c28e22 | ||
|
|
875b4ee5ce | ||
|
|
9ab85ef92e | ||
|
|
89475bb0cf | ||
|
|
4f1e92160a | ||
|
|
fe9533c186 | ||
|
|
70e70eda5a | ||
|
|
cb7a444a56 | ||
|
|
53addd73c3 | ||
|
|
b6c174a324 | ||
|
|
5b75e866d0 | ||
|
|
d0fb384f00 | ||
|
|
c114b786e8 | ||
|
|
e96a7ada50 | ||
|
|
eec35a7c0c | ||
|
|
3b700886f2 | ||
|
|
a397492503 | ||
|
|
dae2aa0be1 | ||
|
|
d064c41057 | ||
|
|
393f6902b3 | ||
|
|
015304f4a3 | ||
|
|
c55672ce5e | ||
|
|
7add29cffe | ||
|
|
c1b8b9d115 | ||
|
|
8717ec76d4 | ||
|
|
e1b9f3e095 | ||
|
|
c40a79e601 | ||
|
|
b75495594a | ||
|
|
24b7740b80 | ||
|
|
aa40b107c9 | ||
|
|
d69fd304cc | ||
|
|
4a57d8c90c | ||
|
|
b8dbee28cc | ||
|
|
2bcc34a59b | ||
|
|
e282e08917 | ||
|
|
b0baa17836 | ||
|
|
ae26376a90 | ||
|
|
58f45a1b12 | ||
|
|
c1a1efa62a | ||
|
|
5b9e9d872d | ||
|
|
13e2fe14aa | ||
|
|
4bfa94dfef | ||
|
|
10852161c9 | ||
|
|
669589b76b | ||
|
|
c35cc1d9e4 | ||
|
|
3cf4fbaf3d | ||
|
|
7eb7b9fa62 | ||
|
|
ec3a65fa8d | ||
|
|
5906e18028 | ||
|
|
0871fa7cc4 | ||
|
|
cdd428a7d8 | ||
|
|
514ffc30da | ||
|
|
df44d21c82 | ||
|
|
dcbdc70919 | ||
|
|
84e50beb65 | ||
|
|
308dbb94b8 | ||
|
|
c642471c16 | ||
|
|
24cfc18383 | ||
|
|
f29b19aff2 | ||
|
|
855f4471e2 | ||
|
|
6ebad21cbe | ||
|
|
a222db8c69 | ||
|
|
66fe7c8a0d | ||
|
|
7623279efe | ||
|
|
570b66230c | ||
|
|
e5d9753368 | ||
|
|
6adaf44f65 | ||
|
|
66e89b71a2 | ||
|
|
32c5df952c | ||
|
|
4363c371c5 | ||
|
|
1b01d065ac | ||
|
|
d3ee49d6f3 | ||
|
|
ded92a8137 | ||
|
|
20614822df | ||
|
|
9740238ec9 | ||
|
|
a2313057c0 | ||
|
|
2b14f166de | ||
|
|
d8207358b0 | ||
|
|
c6ae0fc405 | ||
|
|
0128b962ab | ||
|
|
271de0108d | ||
|
|
c93feeb30f | ||
|
|
7f33c8b499 | ||
|
|
18c1735f7c | ||
|
|
2a71a8ff76 | ||
|
|
e73aae3686 | ||
|
|
3eef84d979 | ||
|
|
60358d01e9 | ||
|
|
05418b4fe5 | ||
|
|
2779f5312b | ||
|
|
e2380404a1 | ||
|
|
998d8f7370 | ||
|
|
5dc3a8e17f | ||
|
|
768f637bd0 | ||
|
|
ee0f46b78f | ||
|
|
84a14b08d5 | ||
|
|
fabfdf04f7 | ||
|
|
966093d8d0 | ||
|
|
4d47ff5ce7 | ||
|
|
8e32583089 | ||
|
|
6019c6d514 | ||
|
|
054754482c | ||
|
|
f7129fdb41 | ||
|
|
f7abd05981 | ||
|
|
514cf23aef | ||
|
|
95b509e0d2 | ||
|
|
83e135f101 | ||
|
|
30b86ea602 | ||
|
|
32a99144f6 | ||
|
|
3544d644ba | ||
|
|
a0ce37fb2c | ||
|
|
a24a1d72ad | ||
|
|
02c7f8e052 | ||
|
|
53b0e8d318 | ||
|
|
c777fbae8f | ||
|
|
98719a1bd5 | ||
|
|
72bcf20870 | ||
|
|
b47668421f | ||
|
|
2239fb253d | ||
|
|
7da4deb3f1 | ||
|
|
b461b005e9 | ||
|
|
75e3a26336 | ||
|
|
5b455cecfe | ||
|
|
faa2005538 | ||
|
|
6ea23b6731 | ||
|
|
db7cd273f0 | ||
|
|
12c80d7c2e | ||
|
|
649ed21d55 | ||
|
|
23cbf9294a | ||
|
|
03b7fcd7bb | ||
|
|
7b8f5e7805 | ||
|
|
798d73eb11 | ||
|
|
df13a83a91 | ||
|
|
61a3bd996d | ||
|
|
b8158affa7 | ||
|
|
ebb4f4af18 | ||
|
|
27a956448c | ||
|
|
0bbdae1d9d | ||
|
|
4c7056de42 | ||
|
|
ae4b86e333 | ||
|
|
8f1e5f2547 | ||
|
|
60518f532d | ||
|
|
cdd17eaae8 | ||
|
|
3293f6ddcd | ||
|
|
1ada760a64 | ||
|
|
d68e241724 | ||
|
|
be0226b624 | ||
|
|
acb720d2bf | ||
|
|
f6bda12a32 | ||
|
|
4b8b5c7329 | ||
|
|
3fa41dfe1a | ||
|
|
4f51393b72 | ||
|
|
9a12f2851f | ||
|
|
98fbc490a4 | ||
|
|
049c3b8c11 | ||
|
|
813f3cae87 | ||
|
|
77c1b50019 | ||
|
|
6bcc4eab24 | ||
|
|
692b0f0160 | ||
|
|
897a5efe7c | ||
|
|
98f4ff6163 | ||
|
|
ec4fe14457 | ||
|
|
75eb2bb534 | ||
|
|
18521ba7af | ||
|
|
a9a0f7b4e0 | ||
|
|
f5ccbb2700 | ||
|
|
0d1bbe967d | ||
|
|
ff4dc71ce4 | ||
|
|
84737f4cb2 | ||
|
|
38d666f223 | ||
|
|
5cfc4ff4a7 | ||
|
|
14a12cc7b6 | ||
|
|
44db32c31b | ||
|
|
12168ba4da | ||
|
|
72aca552e1 | ||
|
|
9ae30a7bbb | ||
|
|
e362c800a4 | ||
|
|
c7468656af | ||
|
|
598b0d8193 | ||
|
|
c27fad5c6e | ||
|
|
85826c633d | ||
|
|
9f7e898ecf | ||
|
|
74ecda8ea2 | ||
|
|
e61141f596 | ||
|
|
bc960c51db | ||
|
|
07066fe7a0 | ||
|
|
974177a074 | ||
|
|
0422893873 | ||
|
|
1d3029fa1e | ||
|
|
af66f72931 | ||
|
|
233bb23bbc | ||
|
|
e031758e4a | ||
|
|
122f4dd727 | ||
|
|
e0261cdfc1 | ||
|
|
6c1f7cc7f5 | ||
|
|
71afdf3ad2 | ||
|
|
ffa63d25a0 | ||
|
|
76c7ddfc52 | ||
|
|
39365c62c9 | ||
|
|
a4713e2d28 | ||
|
|
8bdfddab4a | ||
|
|
a4c0cee251 | ||
|
|
bb8383b065 | ||
|
|
da0f8f1cfd | ||
|
|
62b211bb60 | ||
|
|
3d1465606b | ||
|
|
1d8d726758 | ||
|
|
b0d6684d37 | ||
|
|
56eb0b792c | ||
|
|
9c15610f70 | ||
|
|
91fb323ce0 | ||
|
|
8da56ead81 | ||
|
|
7bbba2e39e | ||
|
|
08e9d15817 | ||
|
|
1b624c3ec7 | ||
|
|
7e2d947f5f | ||
|
|
1e833feb39 | ||
|
|
62cfb45789 | ||
|
|
16b8f591ed | ||
|
|
15361c661f | ||
|
|
cc3f7e8e01 | ||
|
|
d5e8458340 | ||
|
|
b133a8949d | ||
|
|
33131dcbfe | ||
|
|
2ae7c53376 | ||
|
|
4055e0094b | ||
|
|
ee71ba4dea | ||
|
|
b9c24a7b42 | ||
|
|
744bfec4a0 | ||
|
|
8bff438e5e | ||
|
|
7ecf7ba356 | ||
|
|
737bce9c94 | ||
|
|
ec191247a0 | ||
|
|
a9edd2951c | ||
|
|
015334020a | ||
|
|
a3c098a5c6 | ||
|
|
2fd165f5af | ||
|
|
db1151c773 | ||
|
|
02f78c9949 | ||
|
|
192633edb8 | ||
|
|
e5e8ff5783 | ||
|
|
c3ff18de4e | ||
|
|
3ed9abf658 | ||
|
|
552f97c919 | ||
|
|
65fbc1e6f8 | ||
|
|
cbe8528f91 | ||
|
|
5497d362dd | ||
|
|
6b32e07164 | ||
|
|
f7814853f9 | ||
|
|
58aacaad2d | ||
|
|
81672b183d | ||
|
|
76e214a8b9 | ||
|
|
acf02d52ff | ||
|
|
a7acee3eed | ||
|
|
d2645a27e9 | ||
|
|
7d39510450 | ||
|
|
f38ac87ac9 | ||
|
|
d8de0a8853 | ||
|
|
45d0348044 | ||
|
|
8c5af3e4a3 | ||
|
|
ece607e215 | ||
|
|
86a05babed | ||
|
|
1ec73915b8 | ||
|
|
f81e6147ae | ||
|
|
0a4c35f731 | ||
|
|
4f9227a186 | ||
|
|
fb8ea0e683 | ||
|
|
a6c70803ab | ||
|
|
9bd1c84e0f | ||
|
|
1f3fb52470 | ||
|
|
bb6e472939 | ||
|
|
93f33e2305 | ||
|
|
abdf85aa4f | ||
|
|
91686fdb81 | ||
|
|
d2a868f99f | ||
|
|
a143678fd8 | ||
|
|
9914459830 | ||
|
|
b7159d71a6 | ||
|
|
e0c7f36ca9 | ||
|
|
9120f94aa4 | ||
|
|
1900b40192 | ||
|
|
6cbdb3712e | ||
|
|
2a0c428ef9 | ||
|
|
a5d1f7dde4 | ||
|
|
95dd64c774 | ||
|
|
81fa009bf4 | ||
|
|
05ff90e3c1 | ||
|
|
aaffc19232 | ||
|
|
479ac71050 | ||
|
|
4f6b389a37 | ||
|
|
45f5e27526 | ||
|
|
dd8ba97989 | ||
|
|
ab22efd8b1 | ||
|
|
54aee345fc | ||
|
|
8639b373d2 | ||
|
|
5fc69444f6 | ||
|
|
2fe9d678fc | ||
|
|
58b4f8b6bc | ||
|
|
bf667a602d | ||
|
|
9a8b3870ba | ||
|
|
24c23a1d06 | ||
|
|
a548a61e86 | ||
|
|
bc28606d45 | ||
|
|
7f0eb9c51d | ||
|
|
690aa662a8 | ||
|
|
8bd0b5db15 | ||
|
|
b4fd2cc210 | ||
|
|
1a021052fe | ||
|
|
cf738cdb90 | ||
|
|
7a2af23764 | ||
|
|
daaa1962bd | ||
|
|
df878ba1f7 | ||
|
|
1e01868d92 | ||
|
|
9d04ae7edd | ||
|
|
16df7d0b7a | ||
|
|
cd1d2ff447 | ||
|
|
faf6a23d91 | ||
|
|
2f4e81f3b0 | ||
|
|
4182f8fbfb | ||
|
|
c3a2ffde67 | ||
|
|
6502ba6ad6 | ||
|
|
f6c0520f26 | ||
|
|
c912bcab6e | ||
|
|
63c0b445d7 | ||
|
|
653ff61662 | ||
|
|
a7cd0a7807 | ||
|
|
7271cf151b | ||
|
|
82d0e125b2 | ||
|
|
a4c24a1979 | ||
|
|
cff69893a3 | ||
|
|
047237c911 | ||
|
|
449cda1b00 | ||
|
|
a6e53a4858 | ||
|
|
4993f1b1d7 | ||
|
|
199f6b48b1 | ||
|
|
4636261435 | ||
|
|
4e8fdfdf04 | ||
|
|
d00926e089 | ||
|
|
bb15aab50e | ||
|
|
6a4718b712 | ||
|
|
d5adf9b72b | ||
|
|
7365890298 | ||
|
|
0862f97972 | ||
|
|
45439fd742 | ||
|
|
0184e2a5b8 | ||
|
|
50c229f905 | ||
|
|
69b845c0f7 | ||
|
|
caafd9306e | ||
|
|
0d22776b20 | ||
|
|
1803d25c3a | ||
|
|
d56c576e9a | ||
|
|
fbc6d6330f | ||
|
|
fb0cdabe25 | ||
|
|
0474a2a3cb | ||
|
|
180a7e7b80 | ||
|
|
be94d2689e | ||
|
|
b225c7bb4c | ||
|
|
1d44288514 | ||
|
|
af2d2a2c14 | ||
|
|
59d77dd457 | ||
|
|
23d66187a8 | ||
|
|
4476e58080 | ||
|
|
58de6aa10a | ||
|
|
cdd3f3fd4f | ||
|
|
21d6583b5e | ||
|
|
bb46a6ad0f | ||
|
|
f8afaefba0 | ||
|
|
1e38c42646 | ||
|
|
1241ee7041 | ||
|
|
2dc384d641 | ||
|
|
e4dbb49d2f | ||
|
|
02d9d7b053 | ||
|
|
7fc0ea5478 | ||
|
|
8be1c2ab0d | ||
|
|
ff9386dccc | ||
|
|
bba9081ca1 | ||
|
|
65cd7b4787 | ||
|
|
a78239b2aa | ||
|
|
2ac0c4686a | ||
|
|
5233f7584d | ||
|
|
8858dba25e | ||
|
|
9254180cab | ||
|
|
94f9aa298c | ||
|
|
ee9fcaba20 | ||
|
|
43f2d6020e | ||
|
|
6250b1274b | ||
|
|
7f04d12047 | ||
|
|
e88ddab241 | ||
|
|
06f70e11ee | ||
|
|
61cec66ba8 | ||
|
|
5e2fadce15 | ||
|
|
882e75c954 |
@@ -4,3 +4,5 @@
|
|||||||
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
|
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
|
||||||
# interrupted release build cannot commit a twenty-megabyte tar by accident.
|
# interrupted release build cannot commit a twenty-megabyte tar by accident.
|
||||||
/internal/image/builder.tar.placeholder
|
/internal/image/builder.tar.placeholder
|
||||||
|
/cmd/mesh-host/mesh-host
|
||||||
|
/cmd/mesh-bootstrap/mesh-bootstrap
|
||||||
|
|||||||
@@ -171,26 +171,6 @@ carries. The claim is not. The foundation is four container services, and:
|
|||||||
So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly
|
So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly
|
||||||
beats shipping a host that claims a foundation it has never raised.
|
beats shipping a host that claims a foundation it has never raised.
|
||||||
|
|
||||||
## mesh-cli's socket (novox/hq ADR 0272)
|
|
||||||
|
|
||||||
The engine serves `/run/nox-mesh-host/cli.sock` for `mesh-cli`, the operator's command (`internal/meshcli`). It
|
|
||||||
reads the asking account from the kernel (`SO_PEERCRED`) and asks the controller on this node's own subject; the
|
|
||||||
controller decides what the line may do, and the engine runs nothing it receives. One line per connection, each
|
|
||||||
connection's deadline set the moment it is accepted, two lines at once per account and eight for the machine,
|
|
||||||
"busy" beyond that.
|
|
||||||
|
|
||||||
**Only a login session can be the terminal.** The engine reads the asking process's cgroup and sends its login
|
|
||||||
session's scope (`session-<id>.scope` under the account's own `user-<uid>.slice`, which logind's own record in
|
|
||||||
`/run/systemd/sessions/<id>` must say is this account's, of class `user`), or nothing, read the moment the
|
|
||||||
connection is accepted; the controller makes
|
|
||||||
nothing else the terminal. A service running as the operator — the tool runner, a unit of the account's user
|
|
||||||
manager — is at most an ordinary call.
|
|
||||||
|
|
||||||
**A process outside the machine's own namespaces is refused**, whatever its user id: the peer's mount and user
|
|
||||||
namespace must be the engine's. So `mesh-cli` run inside a flatpak, a toolbox or distrobox container, or any other
|
|
||||||
container — even one running as the operator's own id — is refused with the reason; run it from the machine's own
|
|
||||||
shell.
|
|
||||||
|
|
||||||
## A capability is detected, never assumed
|
## A capability is detected, never assumed
|
||||||
|
|
||||||
The reason this is the first thing built rather than a detail of it.
|
The reason this is the first thing built rather than a detail of it.
|
||||||
|
|||||||
@@ -0,0 +1,212 @@
|
|||||||
|
// mesh-compose is the node-engine's composer command (novox/hq ADR 0482 clause 4, ADR 0593 §7, issue 1621): it reads a
|
||||||
|
// composer.Request on standard input, reads every manifest, composes the machines named in each case and writes one
|
||||||
|
// composer.Verdict on standard output. Built at a commit of the node-engine's repository, it answers what that
|
||||||
|
// commit's composition answers, so a group that changes the node-engine's composition is judged by its own next code.
|
||||||
|
//
|
||||||
|
// mesh-compose < request.json > verdict.json
|
||||||
|
// mesh-compose -request request.json > verdict.json
|
||||||
|
// mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
|
||||||
|
//
|
||||||
|
// -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
|
||||||
|
// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
|
||||||
|
// compose; -declarations answers each machine's declared paths with their owners and modes.
|
||||||
|
//
|
||||||
|
// Run as a pipeline step it reads its input, a composer.Request at depth compose, from the file named by the environment
|
||||||
|
// variable MESH_RUN_INPUT and answers the run's output as its composer.Verdict on the last line of standard output
|
||||||
|
// (design 47 G A-167).
|
||||||
|
//
|
||||||
|
// It opens no network connection, bus, store or file beyond its standard input, the -request file, the file
|
||||||
|
// MESH_RUN_INPUT names and the -catalogue directory, and reads no other environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
|
||||||
|
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/composer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodes []string
|
||||||
|
|
||||||
|
func (n *nodes) String() string { return strings.Join(*n, ",") }
|
||||||
|
func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
|
||||||
|
|
||||||
|
// run is the command: its arguments, standard input, output and error; it answers the exit code.
|
||||||
|
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||||
|
fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
|
||||||
|
var machines nodes
|
||||||
|
fs.Var(&machines, "node", "a machine to compose (repeatable)")
|
||||||
|
depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
|
||||||
|
declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
|
||||||
|
requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
var raw []byte
|
||||||
|
var err error
|
||||||
|
// A pipeline step hands its input as a file named by MESH_RUN_INPUT (design 47 G A-167); -request, when given, wins.
|
||||||
|
runInput := os.Getenv("MESH_RUN_INPUT")
|
||||||
|
if *requestFile == "" && runInput != "" {
|
||||||
|
*requestFile = runInput
|
||||||
|
}
|
||||||
|
if *requestFile != "" {
|
||||||
|
raw, err = os.ReadFile(*requestFile)
|
||||||
|
} else {
|
||||||
|
raw, err = io.ReadAll(stdin)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
var req composer.Request
|
||||||
|
if strings.TrimSpace(string(raw)) == "" {
|
||||||
|
if *catalogueDir == "" {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
req = composer.Request{V: composer.Version}
|
||||||
|
} else if req, err = composer.Decode(raw); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if *catalogueDir != "" {
|
||||||
|
read, err := readCatalogue(*catalogueDir)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if len(req.Cases) == 0 {
|
||||||
|
req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
|
||||||
|
}
|
||||||
|
if req.Cases[0].Manifests == nil {
|
||||||
|
req.Cases[0].Manifests = map[string]composer.Manifest{}
|
||||||
|
}
|
||||||
|
for name, m := range read {
|
||||||
|
if _, given := req.Cases[0].Manifests[name]; !given {
|
||||||
|
req.Cases[0].Manifests[name] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No list of predefined keys is read: the composer judges no manifest (novox/hq ADR 0578 §5).
|
||||||
|
}
|
||||||
|
req.Machines = append(req.Machines, machines...)
|
||||||
|
if *depth != "" {
|
||||||
|
req.Depth = *depth
|
||||||
|
}
|
||||||
|
if *declarations {
|
||||||
|
req.Declarations = true
|
||||||
|
}
|
||||||
|
v, err := runWithSeats(req)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
enc := json.NewEncoder(stdout)
|
||||||
|
if err := enc.Encode(v); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoSeatList is what a machine reads, not judged, when the request carries no seat list (novox/hq ADR 0636 §1).
|
||||||
|
const NoSeatList = "the composer was handed no seat list, so it composes nothing (a reader of the seat list acts on nothing " +
|
||||||
|
"new until it has read the list, novox/hq ADR 0636 §1): the asker gives the seat definitions it judges against in the request's seats"
|
||||||
|
|
||||||
|
// runWithSeats composes req against the seat list the request carries (Request.Seats: the definitions the asker's seat list
|
||||||
|
// reader holds, the members' heads' laid over them in a check, ADR 0636 §4), over the foundation's bootstrapped ones
|
||||||
|
// (seatlist.Bootstrapped), as the node-engine composes against the list it read from the bus (ADR 0635 §1). The command
|
||||||
|
// compiles no seat list: without one in the request it reads every manifest and judges no machine, each said not judged
|
||||||
|
// with NoSeatList (ADR 0593 §9), never a refusal or a pass made without the seats' paths, deliveries and holders.
|
||||||
|
func runWithSeats(req composer.Request) (composer.Verdict, error) {
|
||||||
|
if len(req.Seats) == 0 {
|
||||||
|
machines := req.Machines
|
||||||
|
req.Machines = nil
|
||||||
|
v, err := composer.Run(req)
|
||||||
|
if err != nil || len(machines) == 0 {
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
for i := range v.Cases {
|
||||||
|
v.Cases[i].Machines = map[string]composer.Machine{}
|
||||||
|
for _, node := range machines {
|
||||||
|
// Blamed is always written, blaming none here (design 47 G §721, A-469h).
|
||||||
|
v.Cases[i].Machines[node] = composer.Machine{NotJudged: NoSeatList, Blamed: []struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Problem string `json:"problem"`
|
||||||
|
}{}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
defs := map[string]seatdef.Definition{}
|
||||||
|
for _, d := range seatlist.Bootstrapped() {
|
||||||
|
defs[d.Seat] = d
|
||||||
|
}
|
||||||
|
for _, raw := range req.Seats {
|
||||||
|
d, err := seatdef.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return composer.Verdict{}, fmt.Errorf("a seat definition of the request: %w", err)
|
||||||
|
}
|
||||||
|
defs[d.Seat] = d
|
||||||
|
}
|
||||||
|
catalogue.UseSeats(catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))))
|
||||||
|
return composer.Run(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
|
||||||
|
// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
|
||||||
|
func readCatalogue(dir string) (map[string]composer.Manifest, error) {
|
||||||
|
paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
|
||||||
|
paths = append(paths, filepath.Join(dir, "module.json"))
|
||||||
|
}
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
out := map[string]composer.Manifest{}
|
||||||
|
for _, p := range paths {
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var probe struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
name := filepath.Base(filepath.Dir(p))
|
||||||
|
if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
|
||||||
|
name = probe.Module
|
||||||
|
}
|
||||||
|
if _, twice := out[name]; twice {
|
||||||
|
return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
|
||||||
|
}
|
||||||
|
out[name] = composer.Manifest{Raw: raw}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,509 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets/bucketstest"
|
||||||
|
"go/build"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||||
|
"github.com/novox/mesh-host/internal/beside"
|
||||||
|
"github.com/novox/mesh-host/internal/compose"
|
||||||
|
"github.com/novox/mesh-host/internal/composer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
|
||||||
|
func captured(t *testing.T) map[string]composer.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(read) < 100 {
|
||||||
|
t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
|
||||||
|
}
|
||||||
|
return read
|
||||||
|
}
|
||||||
|
|
||||||
|
// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
|
||||||
|
// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
|
||||||
|
// desktop's modules, ace a module whose requirement is provided elsewhere.
|
||||||
|
func meshOf() compose.Input {
|
||||||
|
return compose.Input{Nodes: []compose.Node{
|
||||||
|
{Name: "anchor", Assigned: []string{"nats", "postgres"}},
|
||||||
|
{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
|
||||||
|
{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func request(t *testing.T, depth string) composer.Request {
|
||||||
|
t.Helper()
|
||||||
|
r := composer.Request{V: composer.Version, Depth: depth, HoldingsFromClaims: true,
|
||||||
|
Machines: []string{"anchor", "laptop", "ace"}, Seats: seats(t),
|
||||||
|
Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||||
|
return withInput(t, r, meshOf())
|
||||||
|
}
|
||||||
|
|
||||||
|
// seats is the seat list a request carries, as the checker hands the one its reader holds (ADR 0635 §1, ADR 0636 §4).
|
||||||
|
func seats(t *testing.T) []json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
var out []json.RawMessage
|
||||||
|
for _, d := range seatlisttest.Definitions() {
|
||||||
|
raw, err := json.Marshal(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out = append(out, raw)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withInput is r with in as its input, as a caller puts it on the wire: the request's shape carries the input as JSON.
|
||||||
|
func withInput(t *testing.T, r composer.Request, in compose.Input) composer.Request {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &r.Input); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
|
||||||
|
t.Helper()
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
code := run(args, bytes.NewReader(in), &out, &errOut)
|
||||||
|
var v composer.Verdict
|
||||||
|
if code == 0 {
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||||
|
t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return code, v, errOut.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func encode(t *testing.T, r composer.Request) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
|
||||||
|
// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
|
||||||
|
// refused.
|
||||||
|
func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
|
||||||
|
r := request(t, composer.DepthResolve)
|
||||||
|
r.Cases[0].Manifests["later"] = composer.Manifest{Raw: json.RawMessage(`{"module":"later","version":"1","shares":["x"]}`)}
|
||||||
|
r.Cases[0].Manifests["refused"] = composer.Manifest{Raw: json.RawMessage(`{"module":"refused","version":"1"}`)}
|
||||||
|
r.Cases[0].Verdicts = bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"refused": {V: 1, Module: "refused", Judge: "src2:j", State: meshcatalogclient.VerdictRefused, Problems: []string{"it is wrong"}}})
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
c := v.Cases[0]
|
||||||
|
// ADR 0578 §5, §6: the composer judges nothing; it sets an unknown field aside and names it, and holds a manifest
|
||||||
|
// its verdict refuses, named in the indexer's words.
|
||||||
|
if got := c.Manifests["later"]; !got.Accepted || !strings.Contains(got.SetAside, `"shares"`) {
|
||||||
|
t.Errorf("the manifest with an unknown field reads %+v, want accepted naming the field set aside", got)
|
||||||
|
}
|
||||||
|
if got := c.Manifests["refused"]; got.Accepted || !strings.Contains(got.Refused, "refused the manifest of refused: it is wrong") {
|
||||||
|
t.Errorf("the refused manifest reads %+v, want refused in the verdict's words", got)
|
||||||
|
}
|
||||||
|
if got := c.Manifests["nats"]; !got.Accepted {
|
||||||
|
t.Errorf("nats reads %+v, want accepted", got)
|
||||||
|
}
|
||||||
|
if len(c.Machines) != 3 {
|
||||||
|
t.Fatalf("machines = %v, want the three named", c.Machines)
|
||||||
|
}
|
||||||
|
for name, m := range c.Machines {
|
||||||
|
if !m.Composes && m.Reason == "" {
|
||||||
|
t.Errorf("%s does not compose and says no reason", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1966, the group try of 2026-11-06 10:07: the composer judges against the seat list its request carries, the
|
||||||
|
// node-engine's from the bus (ADR 0635 §1). The uplink's holder steps back from the resolver file for the resolver's
|
||||||
|
// holder by the seat's definition; without the list nothing is judged (ADR 0636 §1), never "both declare the path".
|
||||||
|
resolverMachine := func(withSeats bool) composer.Machine {
|
||||||
|
r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, HoldingsFromClaims: true,
|
||||||
|
Machines: []string{"laptop"}, Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||||
|
if withSeats {
|
||||||
|
r.Seats = seats(t)
|
||||||
|
}
|
||||||
|
r = withInput(t, r, compose.Input{Nodes: []compose.Node{{Name: "laptop",
|
||||||
|
Assigned: []string{"networkmanager", "systemd-resolved", "pacman", "systemd", "dnsmasq"},
|
||||||
|
Capabilities: map[string]bool{"file-references": true, "privileged": true, "service-manager": true, "package-manager": true, "uplink-networkmanager": true}}}})
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 || len(v.Cases) != 1 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
return v.Cases[0].Machines["laptop"]
|
||||||
|
}
|
||||||
|
if m := resolverMachine(true); strings.Contains(m.Reason+strings.Join(m.Problems, " "), "both declare") || m.NotJudged != "" {
|
||||||
|
t.Errorf("with the seat list, the uplink's and the resolver's holders on one machine: %+v, want the uplink's stepping back", m)
|
||||||
|
}
|
||||||
|
if m := resolverMachine(false); m.Composes || m.NotJudged != NoSeatList || m.Reason != "" || m.Blamed == nil {
|
||||||
|
t.Errorf("without a seat list: %+v, want not judged, saying why, blaming none", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Design 47 G §721, §733 (the node-engine's composer): a machine's verdict names the modules it blames, each with its
|
||||||
|
// problem, and always writes blamed (A-469h). On a node composed as its node-engine composes it (ComposedHow), blamed is
|
||||||
|
// the modules the change refuses that the node held neither in its read nor composing without the change (A-469j).
|
||||||
|
file := func(module, path, content string) string {
|
||||||
|
return `{"module": "` + module + `", "version": "1", "resources": [{"id": "f", "type": "file", "path": "` + path +
|
||||||
|
`", "mode": "0600", "content": "` + content + `"}]}`
|
||||||
|
}
|
||||||
|
plain := func(module string) string { return `{"module": "` + module + `", "version": "1"}` }
|
||||||
|
secret := `N=${secret:repository}\\n`
|
||||||
|
search := map[string]string{"m1": plain("m1"), "m2": file("m2", "/var/lib/m2/env", secret), "m3": plain("m3")}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
manifests map[string]string
|
||||||
|
assigned []string
|
||||||
|
how *compose.ComposedHow
|
||||||
|
tries int // the request's blame_tries
|
||||||
|
blamed []blame // each module blamed, its problem holding the words given
|
||||||
|
}{
|
||||||
|
// G-24 H-AC1: a fixture module whose manifest is refused on the machine is blamed by name.
|
||||||
|
{"a module the change refuses is blamed by name", map[string]string{"app": file("app", "/var/lib/app/env", `N=${secret:repository}\\n`)},
|
||||||
|
[]string{"app"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}}, 20, []blame{{Module: "app", Problem: "repository"}}},
|
||||||
|
// Held composing without the change (HeldBefore): left out of blamed.
|
||||||
|
{"a module held before the change is not blamed", map[string]string{"app": file("app", "/var/lib/app/env", `N=${secret:repository}\\n`)},
|
||||||
|
[]string{"app"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}, HeldBefore: map[string]string{"app": "held"}}, 0, []blame{}},
|
||||||
|
// G-24 H-AC2 (A-468l): two modules owning one path are both blamed, with that path.
|
||||||
|
{"two modules owning one path are both blamed", map[string]string{"one": file("one", "/etc/shared.conf", "x"), "two": file("two", "/etc/shared.conf", "y")},
|
||||||
|
[]string{"one", "two"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}}, 0,
|
||||||
|
[]blame{{Module: "one", Problem: `one and two both declare the path "/etc/shared.conf"`}, {Module: "two", Problem: `one and two both declare the path "/etc/shared.conf"`}}},
|
||||||
|
// The same, on a node whose input says not how it composes: nothing is held, both owners are blamed from the refusal.
|
||||||
|
{"two path owners, composed without how the node composes", map[string]string{"one": file("one", "/etc/shared.conf", "x"), "two": file("two", "/etc/shared.conf", "y")},
|
||||||
|
[]string{"one", "two"}, nil, 20,
|
||||||
|
[]blame{{Module: "one", Problem: `one and two both declare the path "/etc/shared.conf"`}, {Module: "two", Problem: `one and two both declare the path "/etc/shared.conf"`}}},
|
||||||
|
// G-24 H-AC5: a machine that fails with no module held is searched over the changed modules placed on it (here every
|
||||||
|
// one, the input naming no manifest its builds were made of, A-469k), in assignment order; the search ends at the
|
||||||
|
// first whose hold ends the problem, so m3 is never tried.
|
||||||
|
{"the search ends at the first module whose hold ends the problem", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 20,
|
||||||
|
[]blame{{Module: "m2", Problem: "repository"}}},
|
||||||
|
// A module whose case manifest is the one its build was made of is not changed (A-469k), so never held: m2's
|
||||||
|
// problem is then no changed module's, and none is blamed.
|
||||||
|
{"a module its build was made of is not tried", search, []string{"m1", "m2", "m3"},
|
||||||
|
&compose.ComposedHow{Node: "a", Manifests: map[string]string{"m2": digest(t, search["m2"])}}, 20, []blame{}},
|
||||||
|
// A-470v: a hold that changes the problem but leaves the machine not composing keeps that module held and blamed,
|
||||||
|
// and the search goes on against the retry's problem: both modules the change refuses are blamed.
|
||||||
|
{"every module the change refuses is blamed", map[string]string{"m1": file("m1", "/var/lib/m1/env", secret), "m2": file("m2", "/var/lib/m2/env", secret), "m3": plain("m3")},
|
||||||
|
[]string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 20, []blame{{Module: "m1", Problem: "repository"}, {Module: "m2", Problem: "repository"}}},
|
||||||
|
// G-24 H-AC4: past the request's blame_tries the modules not yet tried are named so.
|
||||||
|
{"past blame_tries the untried are named", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 1,
|
||||||
|
[]blame{{Module: "m2", Problem: composer.NotTried}, {Module: "m3", Problem: composer.NotTried}}},
|
||||||
|
// A-468i: a request with no blame_tries gets no search; the machine fails on the node's own account (H-AC3).
|
||||||
|
{"a request with no blame_tries is not searched", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 0, []blame{}},
|
||||||
|
// A-469j: one search, the composer's, inside the node-engine's composition (ComposeSent): an input that says not how
|
||||||
|
// its node composes is composed without one, so its failure is the node's own.
|
||||||
|
{"a node that says not how it composes is not searched", search, []string{"m1", "m2", "m3"}, nil, 20, []blame{}},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, Machines: []string{"a"}, Seats: seats(t), BlameTries: row.tries,
|
||||||
|
Cases: []composer.Case{{Name: "change", Manifests: map[string]composer.Manifest{}}}}
|
||||||
|
for name, raw := range row.manifests {
|
||||||
|
r.Cases[0].Manifests[name] = composer.Manifest{Raw: json.RawMessage(raw)}
|
||||||
|
}
|
||||||
|
r = withInput(t, r, compose.Input{Nodes: []compose.Node{{Name: "a", Assigned: row.assigned}}, ComposedHow: row.how})
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if code := run(nil, bytes.NewReader(encode(t, r)), &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), `"blamed":`) {
|
||||||
|
t.Errorf("the verdict writes no blamed: %s", out.String())
|
||||||
|
}
|
||||||
|
var v composer.Verdict
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := v.Cases[0].Machines["a"]
|
||||||
|
if held := row.how != nil && row.how.HeldBefore != nil; m.Composes != held {
|
||||||
|
t.Errorf("composes = %v, want %v: %+v", m.Composes, held, m)
|
||||||
|
}
|
||||||
|
if len(m.Blamed) != len(row.blamed) {
|
||||||
|
t.Fatalf("blamed %+v, want %+v (%+v)", m.Blamed, row.blamed, m)
|
||||||
|
}
|
||||||
|
for i, b := range row.blamed {
|
||||||
|
if m.Blamed[i].Module != b.Module || !strings.Contains(m.Blamed[i].Problem, b.Problem) {
|
||||||
|
t.Errorf("blamed %+v, want %+v", m.Blamed, row.blamed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// digest is the content digest of a manifest (compose.ContentDigest).
|
||||||
|
func digest(t *testing.T, raw string) string {
|
||||||
|
t.Helper()
|
||||||
|
d, err := compose.ContentDigest([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// blame is one module a machine's verdict blames (composer.Machine.Blamed).
|
||||||
|
type blame = struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Problem string `json:"problem"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
|
||||||
|
// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
|
||||||
|
func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
|
||||||
|
for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
|
||||||
|
r := request(t, depth)
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("%s: exit %d: %s", depth, code, stderr)
|
||||||
|
}
|
||||||
|
in := meshOf()
|
||||||
|
in.Catalogue = map[string]catalogue.Manifest{}
|
||||||
|
for name, m := range r.Cases[0].Manifests {
|
||||||
|
// A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined).
|
||||||
|
parsed, err := catalogue.Decode(m.Raw)
|
||||||
|
got := v.Cases[0].Manifests[name]
|
||||||
|
if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
|
||||||
|
t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
in.Catalogue[name] = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
|
||||||
|
composed := 0
|
||||||
|
for _, node := range r.Machines {
|
||||||
|
got := v.Cases[0].Machines[node]
|
||||||
|
var want composer.Machine
|
||||||
|
if depth == composer.DepthResolve {
|
||||||
|
if _, err := compose.Resolve(in, node); err != nil {
|
||||||
|
want = composer.Machine{Reason: err.Error()}
|
||||||
|
} else {
|
||||||
|
want = composer.Machine{Composes: true}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
d, problems, err := compose.Compose(in, node)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
want = composer.Machine{Reason: err.Error()}
|
||||||
|
default:
|
||||||
|
want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
want.Reason = "the node-engine's validator refuses the declaration"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// G-24 H-AC3 (§721): a machine that does not compose and blames no module (ace: its own sealing key) fails on the
|
||||||
|
// node's own account, blamed written empty.
|
||||||
|
want.Blamed = []blame{}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
|
||||||
|
}
|
||||||
|
if got.Composes {
|
||||||
|
composed++
|
||||||
|
} else {
|
||||||
|
t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
|
||||||
|
// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
|
||||||
|
func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
|
||||||
|
`"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
in := encode(t, withInput(t, composer.Request{V: composer.Version, Seats: seats(t)}, compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}))
|
||||||
|
code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
m := v.Cases[0].Machines["laptop"]
|
||||||
|
want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
|
||||||
|
if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
|
||||||
|
m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
|
||||||
|
t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A run step hands the command no standard input: the request is read from the file -request names, or from the one
|
||||||
|
// MESH_RUN_INPUT names, and a file that is not there is a request it cannot read.
|
||||||
|
func TestTheRequestIsReadFromAFile(t *testing.T) {
|
||||||
|
file := filepath.Join(t.TempDir(), "request.json")
|
||||||
|
r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{
|
||||||
|
"demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}}
|
||||||
|
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, v, stderr := runJSON(t, []string{"-request", file}, nil)
|
||||||
|
if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
|
||||||
|
t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v)
|
||||||
|
}
|
||||||
|
if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") {
|
||||||
|
t.Errorf("a request file that is not there: exit %d, %q", code, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pipeline step names its input file in MESH_RUN_INPUT: the command reads the request there and the verdict is the
|
||||||
|
// last line of standard output (design 47 G A-167).
|
||||||
|
r.Depth = composer.DepthCompose
|
||||||
|
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_RUN_INPUT", file)
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
||||||
|
v = composer.Verdict{}
|
||||||
|
if err := json.Unmarshal([]byte(lines[len(lines)-1]), &v); err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
|
||||||
|
t.Fatalf("last line is no verdict: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_RUN_INPUT", file+".gone")
|
||||||
|
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "cannot be read") {
|
||||||
|
t.Errorf("a missing input file: exit %d, %q", code, errOut.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
|
||||||
|
func TestTheExitCodesSayWhatFailed(t *testing.T) {
|
||||||
|
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
in string
|
||||||
|
code int
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"not JSON", nil, "{", 1, "not JSON"},
|
||||||
|
{"another version", nil, `{"v":2}`, 1, "version 2"},
|
||||||
|
{"a digest left unfilled", nil, string(unfilled), 1, "digest"},
|
||||||
|
{"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
|
||||||
|
{"nothing to read", nil, "", 1, "no request"},
|
||||||
|
{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
|
||||||
|
{"an argument", []string{"extra"}, "", 2, "unexpected"},
|
||||||
|
} {
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
|
||||||
|
t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownPackages is every package of this repository the command imports, directly or not, with the packages each
|
||||||
|
// imports: what this repository adds to the composition; mesh-sdk's packages are judged in mesh-sdk.
|
||||||
|
func ownPackages(t *testing.T) map[string][]string {
|
||||||
|
t.Helper()
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
root := filepath.Join(filepath.Dir(file), "..", "..")
|
||||||
|
const module = "github.com/novox/mesh-host"
|
||||||
|
out := map[string][]string{}
|
||||||
|
var walk func(path string)
|
||||||
|
walk = func(path string) {
|
||||||
|
if _, done := out[path]; done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
|
||||||
|
pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", path, err)
|
||||||
|
}
|
||||||
|
out[path] = pkg.Imports
|
||||||
|
for _, imp := range pkg.Imports {
|
||||||
|
if strings.HasPrefix(imp, module+"/") {
|
||||||
|
walk(imp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(composer.Package)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard
|
||||||
|
// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types) and catalog key codec (a catalog entry's key, design 47 G A-144), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and
|
||||||
|
// generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or
|
||||||
|
// another process.
|
||||||
|
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
|
||||||
|
forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
|
||||||
|
own := ownPackages(t)
|
||||||
|
if len(own) < 4 {
|
||||||
|
t.Fatalf("the command reaches %d packages of this repository; the walk found too few to judge", len(own))
|
||||||
|
}
|
||||||
|
for path, imports := range own {
|
||||||
|
for _, imp := range imports {
|
||||||
|
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
|
||||||
|
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") {
|
||||||
|
t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp)
|
||||||
|
}
|
||||||
|
for _, f := range forbidden {
|
||||||
|
if imp == f {
|
||||||
|
t.Errorf("%s imports %s", path, imp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
|
||||||
|
// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
|
||||||
|
// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
|
||||||
|
func TestTheCommandRunsWithNoNetwork(t *testing.T) {
|
||||||
|
if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
|
||||||
|
os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
|
unshare, err := exec.LookPath("unshare")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("no unshare on this machine, so no empty network namespace can be made here")
|
||||||
|
}
|
||||||
|
if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
|
||||||
|
t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
|
||||||
|
}
|
||||||
|
r := request(t, composer.DepthResolve)
|
||||||
|
cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
|
||||||
|
cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
|
||||||
|
cmd.Stdin = bytes.NewReader(encode(t, r))
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
cmd.Stdout, cmd.Stderr = &out, &errOut
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
|
||||||
|
}
|
||||||
|
var v composer.Verdict
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
|
||||||
|
t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(v.Cases[0].Machines))
|
||||||
|
for n := range v.Cases[0].Machines {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
t.Logf("composed with no network: %v", names)
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the
|
||||||
|
// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs:
|
||||||
|
// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran
|
||||||
|
// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a
|
||||||
|
// 0700 directory: a composition carries values sealed to this machine.
|
||||||
|
type appliedStore struct{ dir string }
|
||||||
|
|
||||||
|
func appliedStoreFor(statePath string) appliedStore {
|
||||||
|
return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path.
|
||||||
|
var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
|
||||||
|
|
||||||
|
func (s appliedStore) path(module string) (string, error) {
|
||||||
|
if !appliedName.MatchString(module) || strings.Contains(module, "..") {
|
||||||
|
return "", fmt.Errorf("%q is not a module name the applied store takes", module)
|
||||||
|
}
|
||||||
|
return filepath.Join(s.dir, module+".json"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds says whether the store keeps the module's composition.
|
||||||
|
func (s appliedStore) holds(module string) bool {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err = os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Load() (map[string][]byte, error) {
|
||||||
|
entries, err := os.ReadDir(s.dir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
var errs []error
|
||||||
|
for _, e := range entries {
|
||||||
|
module, ok := strings.CutSuffix(e.Name(), ".json")
|
||||||
|
if !ok || e.IsDir() || !appliedName.MatchString(module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(s.dir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[module] = raw
|
||||||
|
}
|
||||||
|
return out, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Keep(module string, raw []byte) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(s.dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(s.dir, "."+module+".*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmp.Name())
|
||||||
|
if _, err := tmp.Write(raw); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp.Name(), path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Drop(module string) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownModuleApplied is the module of the composition the store holds at the fingerprint fp, the engine's own build's
|
||||||
|
// (design 47 G §527, A-149): "" when fp is none or no record holds it.
|
||||||
|
func ownModuleApplied(s appliedStore, fp string) string {
|
||||||
|
if fp == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
all, _ := s.Load()
|
||||||
|
for module, raw := range all {
|
||||||
|
var c compositions.Composed
|
||||||
|
if module != agent.NodeModule && json.Unmarshal(raw, &c) == nil && c.FP == fp && c.Module != "" {
|
||||||
|
return c.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -27,6 +27,7 @@ func TestAnApplyOutlivesTheLinksContext(t *testing.T) {
|
|||||||
// An apply is bounded all the same: a step that hangs (a docker call that never returns) cannot hold the node agent for
|
// An apply is bounded all the same: a step that hangs (a docker call that never returns) cannot hold the node agent for
|
||||||
// ever (the review of issue 1945).
|
// ever (the review of issue 1945).
|
||||||
func TestAnApplyIsBounded(t *testing.T) {
|
func TestAnApplyIsBounded(t *testing.T) {
|
||||||
|
t.Run("the marks of the haves never hold the pass", marksOfTheHavesNeverHoldThePass)
|
||||||
ctx, done := applyContext(context.Background())
|
ctx, done := applyContext(context.Background())
|
||||||
defer done()
|
defer done()
|
||||||
at, ok := ctx.Deadline()
|
at, ok := ctx.Deadline()
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// authorityPath is where this engine keeps the mesh's own certificate as it last applied it (public): beside the
|
||||||
|
// certificate issued for the machine's name, read into its record (novox/hq ADR 0555 §3).
|
||||||
|
func authorityPath(state string) string { return filepath.Join(filepath.Dir(state), "authority.crt") }
|
||||||
|
|
||||||
|
// keepAuthority keeps the mesh's own certificate a composition carries (the resource a module's certificate.authority
|
||||||
|
// asks for, catalogue.AuthorityID), so the record says the authority this machine holds. A composition carrying none
|
||||||
|
// changes nothing.
|
||||||
|
func keepAuthority(state string, c engcomp.Composed) error {
|
||||||
|
for _, raw := range c.Resources {
|
||||||
|
var r struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(raw, &r) != nil || r.Content == "" || !strings.HasSuffix(r.ID, "."+catalogue.AuthorityID()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := authorityPath(state)
|
||||||
|
if was, err := os.ReadFile(path); err == nil && string(was) == r.Content {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(path, []byte(r.Content), 0o644)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/pem"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||||
|
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
|
||||||
|
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
|
||||||
|
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||||
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
|
res := func(id, content string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": id, "type": "file", "path": "/etc/x", "content": content})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
if err := keepAuthority(state, engcomp.Composed{Resources: []json.RawMessage{res("web.conf", "x")}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(authorityPath(state)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("a composition with no authority kept one: %v", err)
|
||||||
|
}
|
||||||
|
if err := keepAuthority(state, engcomp.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
|
||||||
|
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1126, A6 (ADR 0550 §2; the controller ends by its unassignment): a node enrolled with the controller's key
|
||||||
|
// trusts it never, whatever the lease says; before the vault's signers are read it trusts nothing, once read the
|
||||||
|
// vault's key alone, and the record names the vault's alone. Each change tells the record's writer.
|
||||||
|
controllerPub, controller, _ := ed25519.GenerateKey(nil)
|
||||||
|
vaultPub, vault, _ := ed25519.GenerateKey(nil)
|
||||||
|
trust := newMeshTrust()
|
||||||
|
told := 0
|
||||||
|
trust.onChange(func() { told++ })
|
||||||
|
keys := recordKeys(state, trust.Trusts)
|
||||||
|
ctrlFP, vaultFP := meshvaultclient.SignerFingerprint(controllerPub), meshvaultclient.SignerFingerprint(vaultPub)
|
||||||
|
enrol, _ := meshvaultclient.SignContext(meshvaultclient.KindEnrolment)
|
||||||
|
msg := []byte("a reply")
|
||||||
|
if got := keys().Trusts; len(got) != 0 {
|
||||||
|
t.Errorf("before the signers are read the record trusts %v", got)
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(controller, enrol, msg)); err == nil {
|
||||||
|
t.Error("a controller-signed message is accepted")
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(vault, enrol, msg)); err == nil {
|
||||||
|
t.Error("a vault-signed message is accepted before the vault's key is read")
|
||||||
|
}
|
||||||
|
vaultCA := testAuthority(t, vaultPub, vault)
|
||||||
|
keyValue, _ := json.Marshal(meshvaultclient.NewSignerKey(vaultPub, time.Now()))
|
||||||
|
fp, _ := meshvaultclient.AuthorityFingerprint(vaultCA)
|
||||||
|
authValue, _ := json.Marshal(meshvault.SignersByAuthorityValue{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
|
||||||
|
trust.heardSigner(meshvault.SignersByKeyKey(), keyValue, false, func(string) {})
|
||||||
|
trust.heardSigner(meshvault.SignersByAuthorityKey(), authValue, false, func(string) {})
|
||||||
|
if s, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(vault, enrol, msg)); err != nil || s.Name != meshvaultclient.SignerVault {
|
||||||
|
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(controller, enrol, msg)); err == nil {
|
||||||
|
t.Error("a controller-signed message is accepted once the vault's key is read")
|
||||||
|
}
|
||||||
|
if got := keys().Trusts; strings.Contains(strings.Join(got, " "), ctrlFP) || !meshvaultclient.TrustsKey(got, meshvaultclient.SignerVault, vaultFP) ||
|
||||||
|
trust.VaultAuthority() != vaultCA || told != 2 {
|
||||||
|
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
||||||
|
}
|
||||||
|
// A signers' key naming another fingerprint is not trusted.
|
||||||
|
bad, _ := json.Marshal(meshvault.SignersByKeyValue{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
|
||||||
|
trust.heardSigner(meshvault.SignersByKeyKey(), bad, false, func(string) {})
|
||||||
|
if !meshvaultclient.TrustsKey(keys().Trusts, meshvaultclient.SignerVault, vaultFP) {
|
||||||
|
t.Error("a signers' key that does not check replaced the vault's")
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
|
||||||
|
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
|
||||||
|
serving, err := identity.GenerateServingKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
signedBy := func(ca string, key ed25519.PrivateKey) string {
|
||||||
|
parent, _ := meshvaultclient.ParseCertificate(ca)
|
||||||
|
der, err := x509.CreateCertificate(nil, meshvaultclient.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
|
||||||
|
parent, ed25519.PublicKey(servingPub), key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
|
}
|
||||||
|
var asked []any
|
||||||
|
answer := signedBy(vaultCA, vault)
|
||||||
|
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
|
||||||
|
asked = append(asked, verb, args)
|
||||||
|
return json.Marshal(meshvault.SignAnswer{Kind: meshvaultclient.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||||
|
}
|
||||||
|
written := 0
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != meshvault.VerbSign {
|
||||||
|
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
|
||||||
|
}
|
||||||
|
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
|
||||||
|
t.Errorf("a certificate of another authority was kept: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory,
|
||||||
|
// written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path.
|
||||||
|
t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) {
|
||||||
|
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
|
||||||
|
if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") {
|
||||||
|
t.Fatalf("an empty store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
module string
|
||||||
|
takes bool
|
||||||
|
}{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} {
|
||||||
|
if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes {
|
||||||
|
t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(applied.dir)
|
||||||
|
dirInfo, _ := os.Stat(applied.dir)
|
||||||
|
fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json"))
|
||||||
|
if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode())
|
||||||
|
}
|
||||||
|
if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") {
|
||||||
|
t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err)
|
||||||
|
}
|
||||||
|
if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 {
|
||||||
|
t.Fatalf("the store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
|
||||||
|
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
|
||||||
|
t.Helper()
|
||||||
|
tpl := meshvaultclient.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
|
||||||
|
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
|
}
|
||||||
@@ -3,9 +3,17 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
||||||
@@ -39,4 +47,103 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
|||||||
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
||||||
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
||||||
}
|
}
|
||||||
|
// The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs
|
||||||
|
// the module up again (postgres on novox, 2026-10-31).
|
||||||
|
stopped, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted ||
|
||||||
|
got.Words != agent.BackupInterrupted {
|
||||||
|
t.Errorf("a stopped node agent: %+v", got)
|
||||||
|
}
|
||||||
|
if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted {
|
||||||
|
t.Errorf("silence within the bound taken as interrupted: %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
|
||||||
|
// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
|
||||||
|
// reached it is never asked twice.
|
||||||
|
notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
|
||||||
|
"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
answers []error
|
||||||
|
within time.Duration
|
||||||
|
asked int
|
||||||
|
words string
|
||||||
|
}{
|
||||||
|
{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
|
||||||
|
// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
|
||||||
|
{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
|
||||||
|
0, "times over"},
|
||||||
|
{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
|
||||||
|
time.Second, 1, "deadline exceeded"},
|
||||||
|
{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
|
||||||
|
} {
|
||||||
|
asked := 0
|
||||||
|
_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
|
||||||
|
asked++
|
||||||
|
return json.RawMessage(`{}`), row.answers[asked-1]
|
||||||
|
}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
|
||||||
|
if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
|
||||||
|
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
|
||||||
|
// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
|
||||||
|
told := &settledRecorder{}
|
||||||
|
events := make(chan []byte, 4)
|
||||||
|
for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
|
||||||
|
events <- []byte(body)
|
||||||
|
}
|
||||||
|
linked, end := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for len(told.heard()) < 2 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
end()
|
||||||
|
<-done
|
||||||
|
if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
|
||||||
|
t.Errorf("the agent was told %q settled, want distribution,gitea", got)
|
||||||
|
}
|
||||||
|
// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
|
||||||
|
// one (ENG-8, ENG-10).
|
||||||
|
d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
|
||||||
|
`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pull := func(image string) error {
|
||||||
|
return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
|
||||||
|
t.Errorf("a store's image is said as %+v", got)
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
|
||||||
|
t.Errorf("a public image is said as %+v", got)
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
|
||||||
|
t.Errorf("a failure that is no pull is said as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledRecorder is the agent a hearing tells.
|
||||||
|
type settledRecorder struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
list []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *settledRecorder) Settled(holder string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.list = append(r.list, holder)
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *settledRecorder) heard() []string {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return append([]string(nil), r.list...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// A composition that names a file the provisioner fills without saying it is filled would have this node-engine write
|
// A composition that names a file the provisioner fills without saying it is filled would have this node-engine write
|
||||||
@@ -13,7 +12,7 @@ import (
|
|||||||
// when ace, g14 and shanks wrote claude-code's, node-env's and power's files empty).
|
// when ace, g14 and shanks wrote claude-code's, node-env's and power's files empty).
|
||||||
func TestAFileItCannotProduceIsNeverWrittenEmpty(t *testing.T) {
|
func TestAFileItCannotProduceIsNeverWrittenEmpty(t *testing.T) {
|
||||||
unit := json.RawMessage(`{"id":"power.after-boot-unit","type":"file","path":"/etc/systemd/system/mesh-power-after-boot.service","mode":"0644","owner-at-runtime":"owned"}`)
|
unit := json.RawMessage(`{"id":"power.after-boot-unit","type":"file","path":"/etc/systemd/system/mesh-power-after-boot.service","mode":"0644","owner-at-runtime":"owned"}`)
|
||||||
c := delivery.Composed{Node: "ace", Resources: []json.RawMessage{unit}}
|
c := engcomp.Composed{Node: "ace", Resources: []json.RawMessage{unit}}
|
||||||
d, err := declarationOf(c)
|
d, err := declarationOf(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -25,7 +24,7 @@ func TestAFileItCannotProduceIsNeverWrittenEmpty(t *testing.T) {
|
|||||||
if d, _ = declarationOf(c); refuseEmptyWrites(d) != nil {
|
if d, _ = declarationOf(c); refuseEmptyWrites(d) != nil {
|
||||||
t.Fatalf("a filled file was refused: %v", refuseEmptyWrites(d))
|
t.Fatalf("a filled file was refused: %v", refuseEmptyWrites(d))
|
||||||
}
|
}
|
||||||
plain := delivery.Composed{Node: "ace", Resources: []json.RawMessage{json.RawMessage(
|
plain := engcomp.Composed{Node: "ace", Resources: []json.RawMessage{json.RawMessage(
|
||||||
`{"id":"x.f","type":"file","path":"/etc/x","mode":"0644","content":"x"}`)}}
|
`{"id":"x.f","type":"file","path":"/etc/x","mode":"0644","content":"x"}`)}}
|
||||||
if d, _ = declarationOf(plain); refuseEmptyWrites(d) != nil {
|
if d, _ = declarationOf(plain); refuseEmptyWrites(d) != nil {
|
||||||
t.Fatalf("a file with its content was refused: %v", refuseEmptyWrites(d))
|
t.Fatalf("a file with its content was refused: %v", refuseEmptyWrites(d))
|
||||||
|
|||||||
@@ -0,0 +1,457 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||||
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take
|
||||||
|
// would set and what the disk holds now, and whether this machine is root-free.
|
||||||
|
|
||||||
|
// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and
|
||||||
|
// group, its mode, and which file it is.
|
||||||
|
func diskOf(info os.FileInfo) string {
|
||||||
|
owner := "unknown"
|
||||||
|
if uid, gid, ok := apply.OwnerOf(info); ok {
|
||||||
|
owner = fmt.Sprintf("%d:%d", uid, gid)
|
||||||
|
}
|
||||||
|
which := ""
|
||||||
|
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||||
|
which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the
|
||||||
|
// declared owner and mode it will set, and the disk as it is.
|
||||||
|
func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) {
|
||||||
|
return func(path string) (link.Judged, error) {
|
||||||
|
found, info, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, err
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info),
|
||||||
|
Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleOf(id string) string {
|
||||||
|
if at := strings.LastIndex(id, "."); at > 0 {
|
||||||
|
return id[:at]
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this
|
||||||
|
// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is.
|
||||||
|
func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) {
|
||||||
|
return func(act, access string) (link.Judged, error) {
|
||||||
|
d, err := declaredNow(statePath, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err)
|
||||||
|
}
|
||||||
|
for _, l := range d.Libraries {
|
||||||
|
if apply.LibraryAccess(l.Library) != access {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !l.Home {
|
||||||
|
return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+
|
||||||
|
"taken on its home", l.Library)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
|
||||||
|
defer cancel()
|
||||||
|
if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+
|
||||||
|
"(it is judged %s), so there is nothing to take", l.Library, v.Kind)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(l.Root)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err)
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: access, Path: filepath.Clean(l.Root),
|
||||||
|
Sets: "the group " + l.Group + ", group write and setgid on everything under its root",
|
||||||
|
Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil
|
||||||
|
}
|
||||||
|
return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563
|
||||||
|
// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again
|
||||||
|
// by hand.
|
||||||
|
func askTakes(acts *link.HandActs, waiting []string, say func(string)) {
|
||||||
|
if acts == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, access := range waiting {
|
||||||
|
if acts.AskedFor(link.ActAccess, access) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access,
|
||||||
|
"its root holds data outside its group", true); err != nil && say != nil {
|
||||||
|
say("the take of " + access + " could not be asked: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastAccounts is the newest look at every account (the health loop's), which root-free judges from.
|
||||||
|
var lastAccounts atomic.Pointer[accounts.Statement]
|
||||||
|
|
||||||
|
// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor).
|
||||||
|
const rootFreshFor = 15 * time.Minute
|
||||||
|
|
||||||
|
// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most
|
||||||
|
// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such
|
||||||
|
// account declared, and the login shell's execute not served here. Anything else is no pass, and says why.
|
||||||
|
func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict {
|
||||||
|
v := link.RootVerdict{Machine: node, Judged: now.UTC()}
|
||||||
|
var not []string
|
||||||
|
confined := ""
|
||||||
|
switch {
|
||||||
|
case st == nil:
|
||||||
|
not = append(not, "this machine's engine has judged no account yet")
|
||||||
|
case now.Sub(st.At) > rootFreshFor:
|
||||||
|
not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+
|
||||||
|
"minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339),
|
||||||
|
int(rootFreshFor.Minutes())))
|
||||||
|
default:
|
||||||
|
var named []string
|
||||||
|
for _, a := range st.Accounts {
|
||||||
|
if !a.Root {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
named = append(named, a.Name)
|
||||||
|
if a.State != accounts.Healthy {
|
||||||
|
not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(named) == 0 {
|
||||||
|
not = append(not, "no account here is declared never to become root, so agents run as the operator's "+
|
||||||
|
"account, which may become root")
|
||||||
|
} else if len(not) == 0 {
|
||||||
|
confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " +
|
||||||
|
st.At.UTC().Format(time.RFC3339) + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served {
|
||||||
|
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+
|
||||||
|
"can become root ("+servedWhy+")")
|
||||||
|
}
|
||||||
|
if len(not) > 0 {
|
||||||
|
v.Why = strings.Join(not, "; ")
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
v.Free, v.Why = true, confined+"; the login shell's execute is not served here"
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNoneKnownWord(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "is not judged"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's
|
||||||
|
// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery.
|
||||||
|
const settingFileSuffix = ".tools-conf"
|
||||||
|
|
||||||
|
// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served
|
||||||
|
// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied
|
||||||
|
// here says anything but withhold or cannot be read. why says which.
|
||||||
|
func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) {
|
||||||
|
var why []string
|
||||||
|
if heard == nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here")
|
||||||
|
} else if h, err := heard(); err != nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error())
|
||||||
|
} else if h {
|
||||||
|
why = append(why, "the bus hears execute answered here")
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
why = append(why, "what this machine applied could not be read: "+err.Error())
|
||||||
|
}
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if value, err := settingIn(r.Target); err != nil {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error())
|
||||||
|
} else if value != "withhold" {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(why) > 0, strings.Join(why, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingIn is the value of execute in a setting file, "" when it says none.
|
||||||
|
func settingIn(path string) (string, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
value := ""
|
||||||
|
lines := bufio.NewScanner(f)
|
||||||
|
for lines.Scan() {
|
||||||
|
t := strings.TrimSpace(lines.Text())
|
||||||
|
if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" {
|
||||||
|
value = strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
value = "not said"
|
||||||
|
}
|
||||||
|
return value, lines.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8).
|
||||||
|
func keepWaiting(vs []link.ResourceHealth) []string {
|
||||||
|
var out []string
|
||||||
|
for _, v := range vs {
|
||||||
|
for _, w := range v.AccessWaits {
|
||||||
|
if w.Reason == link.WaitKeep {
|
||||||
|
out = append(out, w.Access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's
|
||||||
|
// signed asks, its asks to the operator over the session's link, its acts kept beside the state.
|
||||||
|
func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs {
|
||||||
|
state := link.BusState{Session: queue, Node: node}
|
||||||
|
h := &link.HandActs{Node: node, State: state,
|
||||||
|
JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state),
|
||||||
|
JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||||
|
JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running,
|
||||||
|
RootFree: func() link.RootVerdict {
|
||||||
|
served, why := executeServedHere(opts.state, func() (bool, error) {
|
||||||
|
return queue.HeardServed("node-login-shell", "execute")
|
||||||
|
})
|
||||||
|
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
|
||||||
|
},
|
||||||
|
ManifestReads: queue.StatedReads, Say: queue.SayEngine}
|
||||||
|
asker := occ.EngineAsker(node)
|
||||||
|
h.Asker = &occ.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||||
|
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no
|
||||||
|
// link was open, an ask past its expiry): besides each link's opening.
|
||||||
|
const reconcileEvery = 5 * time.Minute
|
||||||
|
|
||||||
|
func reconcileActs(ctx context.Context, h *link.HandActs) {
|
||||||
|
t := time.NewTicker(reconcileEvery)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
h.Reconcile()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositionReadFor bounds one read of the verb composition: the revisions of one key from the bucket.
|
||||||
|
const compositionReadFor = 20 * time.Second
|
||||||
|
|
||||||
|
// compositionFrom answers the verb composition from the node agent that runs now (issue 774); refused in words while
|
||||||
|
// none does.
|
||||||
|
func compositionFrom(current *agentNow, withheld func(ctx context.Context, module string) ([]link.WithheldSubject, string)) func(module, digest string, diff bool) (any, error) {
|
||||||
|
return func(module, digest string, diff bool) (any, error) {
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no composition yet")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), compositionReadFor)
|
||||||
|
defer cancel()
|
||||||
|
view, err := ag.Composition(ctx, module, digest, diff)
|
||||||
|
if err != nil {
|
||||||
|
return view, err
|
||||||
|
}
|
||||||
|
// And what the grants part withholds of it here, through its verb withheld (design 46 approval mark §6).
|
||||||
|
subjects, unknown := withheld(ctx, module)
|
||||||
|
for _, s := range subjects {
|
||||||
|
view.Withheld = append(view.Withheld, agent.WithheldView{Subject: s.Subject, Why: s.Why})
|
||||||
|
}
|
||||||
|
view.WithheldUnknown = unknown
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeReadFor bounds the verb compose's wait for its pass: a pass that waits on a seat longer says what it waits on.
|
||||||
|
const composeReadFor = 2 * time.Minute
|
||||||
|
|
||||||
|
// composeFrom answers the verb compose (issue 774): a pass of the composer that runs now, and with module that module's
|
||||||
|
// apply tried again by the node agent; refused in words while the node composes nothing of its own.
|
||||||
|
func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Composer]) func(module string) (any, error) {
|
||||||
|
return func(module string) (any, error) {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (it applies the controller's " +
|
||||||
|
"compositions, or its node agent is starting), so there is no pass to make")
|
||||||
|
}
|
||||||
|
out := map[string]any{}
|
||||||
|
if module != "" {
|
||||||
|
// Only a failed apply that is not held is tried again; what runs is never taken away (review round 1: the
|
||||||
|
// verb has no warrant, ADR 0563 §2, and a hold for a failed backup is the operator's to release, ADR 0568 §4).
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so no apply is tried again")
|
||||||
|
}
|
||||||
|
if err := ag.RetryFailed(module); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["retried"] = module
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), composeReadFor)
|
||||||
|
defer cancel()
|
||||||
|
pass, err := c.ComposeNow(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["pass"] = pass
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// packagesReadFor bounds the verb packages' reads and its write.
|
||||||
|
const packagesReadFor = 20 * time.Second
|
||||||
|
|
||||||
|
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages;
|
||||||
|
// with write, the key written again from what the
|
||||||
|
// composer that runs now handed, refused in words while the node composes none of its own.
|
||||||
|
func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) {
|
||||||
|
return func(write bool) (any, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), packagesReadFor)
|
||||||
|
defer cancel()
|
||||||
|
var wrote *bool
|
||||||
|
if write {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (its node agent is starting), " +
|
||||||
|
"so it has no packages of its own to write")
|
||||||
|
}
|
||||||
|
w, err := c.WritePackages(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("this node's key of %s is not written: %w", link.PackagesBucket, err)
|
||||||
|
}
|
||||||
|
wrote = &w
|
||||||
|
}
|
||||||
|
key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node)
|
||||||
|
v := link.PackagesAnswer(node, key, keyErr)
|
||||||
|
v.Wrote = wrote
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeCandidateFrom answers the verb compose-candidate (novox/hq ADR 0593 §7): the check's candidate composed on this
|
||||||
|
// node in shadow by the run's shadow, applying and writing nothing, within the shadow's own bound (shorter than
|
||||||
|
// nec.AnswerBound, selfcompose candidateWithin). The answer is
|
||||||
|
// the seat's shape (nodeengine.ComposeCandidateAnswer), checked as it is encoded; a node whose shadow does not run yet
|
||||||
|
// answers unready, which the checker takes as no answer.
|
||||||
|
func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow]) func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
return func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
ask := nodeengine.ComposeCandidateInput{Candidate: candidate, Input: input, Call: call}
|
||||||
|
if _, err := nec.EncodeAsk(ask); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var a nodeengine.ComposeCandidateAnswer
|
||||||
|
if s := shadow.Load(); s == nil {
|
||||||
|
// Unready, never an error the checker reads words of: its agent is starting (ADR 0593 §9).
|
||||||
|
a = nodeengine.ComposeCandidateAnswer{Node: node, Candidate: candidate, Verdict: nec.Unready,
|
||||||
|
Refused: "this node's shadow does not run now (its node agent is starting), so it composes no candidate"}
|
||||||
|
} else {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), nec.AnswerBound)
|
||||||
|
defer cancel()
|
||||||
|
a = s.ComposeCandidate(ctx, ask)
|
||||||
|
}
|
||||||
|
return answerComposeCandidate(a, candidate, input, call, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstViewsFrom answers the seat's verb first-views from the composer of this run (design 47 G §698): the views the node-engine
|
||||||
|
// follows (none), the entries it keeps and the wants it composes from.
|
||||||
|
func firstViewsFrom(composer *atomic.Pointer[selfcompose.Composer]) func() (any, error) {
|
||||||
|
return func() (any, error) {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (it applies the controller's compositions, or its node agent is starting), so it keeps no entry")
|
||||||
|
}
|
||||||
|
views, entries, wants := c.FirstViews()
|
||||||
|
return map[string]any{"first-views": views, "entries": entries, "wants": wants}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// entryAgainFrom asks one entry of the catalog half again, for the seat's verb entry-again (design 47 G §698).
|
||||||
|
func entryAgainFrom(composer *atomic.Pointer[selfcompose.Composer]) func(ctx context.Context, fingerprint string) error {
|
||||||
|
return func(ctx context.Context, fingerprint string) error {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return errors.New("this node composes none of its own modules now, so it keeps no entry to ask again")
|
||||||
|
}
|
||||||
|
return c.EntryAgain(ctx, fingerprint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerComposeCandidate encodes the answer of compose-candidate. The verb is marked chunks (A-237b, A-237c): asked with input and a
|
||||||
|
// call token, the composition input goes in chunks over the link's connection, whatever its size, and the answer names them;
|
||||||
|
// asked with no token (the checker's, until G removes it, G-6 §321, A-237c) it stays inline in the answer.
|
||||||
|
func answerComposeCandidate(a nodeengine.ComposeCandidateAnswer, candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
var chunked []byte
|
||||||
|
if input && call != "" && a.Input != nil {
|
||||||
|
if line == nil {
|
||||||
|
return nil, errors.New("this engine has no bus connection to send the composition input on")
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(a.Input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the composition input of %s cannot be encoded: %w", candidate, err)
|
||||||
|
}
|
||||||
|
chunked, a.Input = body, nil
|
||||||
|
a.Chunks = int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
||||||
|
}
|
||||||
|
raw, err := nec.EncodeAnswer(a)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("this node's answer to %s does not fit the seat's shape: %w", candidate, err)
|
||||||
|
}
|
||||||
|
if chunked != nil {
|
||||||
|
send := buscalls.SeatChunks{Seat: nodeengine.Seat, Verb: nodeengine.VerbComposeCandidate, Call: call}
|
||||||
|
// The answer names the chunks first; the send runs after it and ends at a chunk the caller does not acknowledge.
|
||||||
|
go func() {
|
||||||
|
if err := send.Send(line, chunked, buscalls.SeatAckWithin, log.Printf); err != nil {
|
||||||
|
log.Printf("the send %s of the composition input of %s: %v", call, candidate, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
return json.RawMessage(raw), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// enrolThroughSeat joins this machine through the seat mesh-node-enroller (novox/hq ADR 0587 §6, issue 774): it redeems the token
|
||||||
|
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
|
||||||
|
// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this
|
||||||
|
// token (link.ErrNoSeat) is said as the error: nothing was sent or kept.
|
||||||
|
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
|
||||||
|
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error {
|
||||||
|
|
||||||
|
req := meshnodeenrollerclient.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
|
||||||
|
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
|
||||||
|
if found != nil {
|
||||||
|
var err error
|
||||||
|
if req.Tunnel, err = json.Marshal(found); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(meshnodeenrollerclient.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
|
||||||
|
to := link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}
|
||||||
|
membership := link.Membership{Node: node, Broker: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}
|
||||||
|
j := link.Joining{Node: node, Signer: token.SignerKey(),
|
||||||
|
Redeem: func(ctx context.Context) (meshnodeenrollerclient.RedeemAnswer, error) {
|
||||||
|
return link.Redeem(ctx, to, node, token.Secret, req, opts.timeout)
|
||||||
|
},
|
||||||
|
Open: func(a meshnodeenrollerclient.RedeemAnswer) (string, error) {
|
||||||
|
value, err := sealing.Open(a.Account)
|
||||||
|
return string(value), err
|
||||||
|
},
|
||||||
|
Keep: func(a meshnodeenrollerclient.RedeemAnswer, password string) error {
|
||||||
|
// The mesh's name for this node wins over what the machine called itself, as on the controller's path.
|
||||||
|
mine.Node = a.Node
|
||||||
|
mine.Membership = identity.Membership{Broker: token.Broker, Fingerprint: token.Fingerprint, Signer: token.Signer,
|
||||||
|
Password: password, Transport: link.OnNATS}
|
||||||
|
if err := writeLogins(loginsFile(opts.state), link.LoginsState{Current: a.Login, Values: map[string]string{a.Login: password}}); err != nil {
|
||||||
|
return fmt.Errorf("cannot keep the login %s: %w", a.Login, err)
|
||||||
|
}
|
||||||
|
if err := identity.Save(identityPath, mine); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !mine.Membership.Joined() {
|
||||||
|
return errors.New("the answer did not say how to reach the mesh again, so the identity could not be used after a restart")
|
||||||
|
}
|
||||||
|
return keepJoinedKeys(opts, mine, sealing, serving)
|
||||||
|
},
|
||||||
|
Dial: func(ctx context.Context, login, password string) (link.Confirming, error) {
|
||||||
|
return link.DialConfirming(ctx, membership, login, password, opts.timeout)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
answer, err := j.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("\nenrolled as %s through the seat mesh-node-enroller, dialled as %s and confirmed\n", answer.Node, answer.Login)
|
||||||
|
fmt.Printf(" identity %s\n", identityPath)
|
||||||
|
fmt.Println(" nothing is applied until the assignment of this node names a module")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
|
||||||
|
// not there looks joined and cannot come up, so the identity is written first and a failure here is said plainly.
|
||||||
|
func keepJoinedKeys(opts options, mine identity.Identity, sealing identity.SealingKey, serving identity.ServingKey) error {
|
||||||
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state), []byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's overlay key: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(identity.SealingKeyPath(opts.state), []byte(sealing.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's sealing key: %w", err)
|
||||||
|
}
|
||||||
|
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's serving key: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -2,7 +2,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -82,6 +84,77 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
|||||||
if err != nil || string(got) != string(body) {
|
if err != nil || string(got) != string(body) {
|
||||||
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
|
||||||
|
// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
|
||||||
|
gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
|
||||||
|
never := []byte(strings.Repeat("never had here\n", 6000))
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
raw []byte
|
||||||
|
onDisk []byte // nil removes the file
|
||||||
|
refused string // "" applies
|
||||||
|
}{
|
||||||
|
{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
|
||||||
|
{"the file here is gone, its object deleted", raw, nil, ""},
|
||||||
|
{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
|
||||||
|
} {
|
||||||
|
if row.onDisk == nil {
|
||||||
|
_ = os.Remove(target)
|
||||||
|
} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
|
||||||
|
if row.refused == "" {
|
||||||
|
got, _ := os.ReadFile(target)
|
||||||
|
if report.Refused != "" || string(got) != string(body) {
|
||||||
|
t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
|
||||||
|
row.name, report.Refused, len(got))
|
||||||
|
}
|
||||||
|
} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
|
||||||
|
t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago,
|
||||||
|
// while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with
|
||||||
|
// no object; a file someone else changed since is left as named.
|
||||||
|
newer := []byte(strings.Repeat("the export the node wrote since\n", 3000))
|
||||||
|
newerName, _ := declaration.ObjectName(declaration.DigestOf(newer))
|
||||||
|
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}})
|
||||||
|
if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Fatalf("924: the newer export: %s", report.Refused)
|
||||||
|
}
|
||||||
|
stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) +
|
||||||
|
`","size":` + strconv.Itoa(len(body)) + `}`)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
edit []byte // what someone else wrote there since; nil for nobody
|
||||||
|
digest string
|
||||||
|
}{
|
||||||
|
{"the file is what the node wrote", nil, declaration.DigestOf(newer)},
|
||||||
|
{"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)},
|
||||||
|
} {
|
||||||
|
if row.edit != nil {
|
||||||
|
if err := os.WriteFile(target, row.edit, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runs := runsAsWritten([]engcomp.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state)
|
||||||
|
var got struct {
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(runs[0].Resources[0], &got)
|
||||||
|
if got.Digest != row.digest {
|
||||||
|
t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest)
|
||||||
|
}
|
||||||
|
if row.edit == nil {
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`)
|
||||||
|
if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
||||||
|
|||||||
+76
-10
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,15 +11,17 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
"github.com/novox/mesh-host/internal/filling"
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
||||||
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
||||||
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
|
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) {
|
||||||
e := fillingEngine(opts, signer, say)
|
e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say)
|
||||||
fillingNow.Store(e)
|
fillingNow.Store(e)
|
||||||
defer fillingNow.CompareAndSwap(e, nil)
|
defer fillingNow.CompareAndSwap(e, nil)
|
||||||
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
||||||
@@ -30,9 +33,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s
|
|||||||
var fillingNow atomic.Pointer[filling.Engine]
|
var fillingNow atomic.Pointer[filling.Engine]
|
||||||
|
|
||||||
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
||||||
// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled).
|
// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled
|
||||||
// What it told, or why it could not, is said.
|
// (agent.Env.Settled, "<module> settled"), the haves were read at the start, or a module was taken as running
|
||||||
func knockFilling(module string, say func(string)) {
|
// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said.
|
||||||
|
func knockFilling(after string, say func(string)) {
|
||||||
e := fillingNow.Load()
|
e := fillingNow.Load()
|
||||||
if e == nil {
|
if e == nil {
|
||||||
return
|
return
|
||||||
@@ -40,16 +44,19 @@ func knockFilling(module string, say func(string)) {
|
|||||||
told, err := e.Knock(filling.Socket)
|
told, err := e.Knock(filling.Socket)
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
say(fmt.Sprintf("after %s settled: %v", module, err))
|
say(fmt.Sprintf("after %s: %v", after, err))
|
||||||
case told != "":
|
case told != "":
|
||||||
say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told))
|
say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
||||||
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
||||||
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
||||||
func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
|
//
|
||||||
|
// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a
|
||||||
|
// file so filled is written 0600 when the declaration gives no mode.
|
||||||
|
func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine {
|
||||||
return &filling.Engine{
|
return &filling.Engine{
|
||||||
Declared: func() (*declaration.Declaration, error) {
|
Declared: func() (*declaration.Declaration, error) {
|
||||||
return declaredNow(opts.state, signer)
|
return declaredNow(opts.state, signer)
|
||||||
@@ -73,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
|
content, secret, err := secrets.fill(ctx, filledModule(d, path), content)
|
||||||
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("%s was not written: %w", path, err)
|
||||||
|
}
|
||||||
|
report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner,
|
||||||
|
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)),
|
||||||
|
apply.Filled{Secret: secret})
|
||||||
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||||
if err := store.Save(opts.state, updated); err != nil {
|
if err := store.Save(opts.state, updated); err != nil {
|
||||||
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
||||||
@@ -93,3 +105,57 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// filledModule is the module whose file the declaration declares at path, or "" when none or several do.
|
||||||
|
func filledModule(d *declaration.Declaration, path string) string {
|
||||||
|
module := ""
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if f, ok := r.(*declaration.File); ok && f.Path == path {
|
||||||
|
if module != "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
module, _ = d.ModuleOfID(f.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return module
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key.
|
||||||
|
func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets {
|
||||||
|
p := provisionSecrets{node: node, open: sealOpener(statePath)}
|
||||||
|
if queue == nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
// What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key.
|
||||||
|
p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
wctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
updates, err := queue.KVWatch(wctx, buckets.Provisions, prefix+">")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case u, ok := <-updates:
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("the watch of %s ended before it caught up", buckets.Provisions)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case u.Caught:
|
||||||
|
return out, nil
|
||||||
|
case u.Deleted:
|
||||||
|
delete(out, u.Key)
|
||||||
|
default:
|
||||||
|
out[u.Key] = u.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
return queue.AskVault(ctx, "own", args)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,9 +7,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/filling"
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
@@ -45,7 +48,34 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
|||||||
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
e := fillingEngine(opts, public, func(string) {})
|
// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
|
||||||
|
// the opener reads the box's "v").
|
||||||
|
provision := map[string]string{}
|
||||||
|
vaultName := "provision/n2/db"
|
||||||
|
secrets := provisionSecrets{node: "n1",
|
||||||
|
list: func(_ context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for k, v := range provision {
|
||||||
|
if strings.HasPrefix(k, prefix) {
|
||||||
|
out[k] = []byte(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
},
|
||||||
|
own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
raw, _ := json.Marshal(args)
|
||||||
|
var a struct{ Credentials []map[string]any }
|
||||||
|
_ = json.Unmarshal(raw, &a)
|
||||||
|
c := a.Credentials[0]
|
||||||
|
c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"}
|
||||||
|
return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
|
||||||
|
},
|
||||||
|
open: func(sealed string) ([]byte, error) {
|
||||||
|
var b struct{ V string }
|
||||||
|
err := json.Unmarshal([]byte(sealed), &b)
|
||||||
|
return []byte(b.V), err
|
||||||
|
}}
|
||||||
|
e := fillingEngine(opts, public, secrets, func(string) {})
|
||||||
uid := uint32(os.Getuid())
|
uid := uint32(os.Getuid())
|
||||||
if err := e.Accepts(uid); err != nil {
|
if err := e.Accepts(uid); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -102,4 +132,75 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
|||||||
if a := <-done; a.Action != "updated" {
|
if a := <-done; a.Action != "updated" {
|
||||||
t.Fatalf("the second write: %+v", a)
|
t.Fatalf("the second write: %+v", a)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
|
||||||
|
// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
|
||||||
|
keyed := func(state string, ref bool, id, local string) string {
|
||||||
|
p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"},
|
||||||
|
Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
|
||||||
|
if state != provisions.Provided {
|
||||||
|
p.Why = "the provider has not answered"
|
||||||
|
}
|
||||||
|
if ref {
|
||||||
|
p.Reference = &provisions.Reference{ID: id, Generation: 2}
|
||||||
|
}
|
||||||
|
raw, err := provisions.Encode(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, state string
|
||||||
|
ref bool
|
||||||
|
content string
|
||||||
|
want string // the file after; "" keeps what was there
|
||||||
|
mode os.FileMode
|
||||||
|
vault string // the vault's credential name, when not the one the provision names
|
||||||
|
refID string // the reference's id, when not the provision's shared value
|
||||||
|
local string // the provision's local name (ADR 0094): the placeholder names it
|
||||||
|
}{
|
||||||
|
{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
|
||||||
|
{name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true,
|
||||||
|
refID: "provision/n9/db", content: "password=${secret:db}\n"},
|
||||||
|
{name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"},
|
||||||
|
{name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true,
|
||||||
|
vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
|
||||||
|
content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
|
||||||
|
{name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin",
|
||||||
|
content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600},
|
||||||
|
{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
|
||||||
|
} {
|
||||||
|
before, _ := os.ReadFile(path)
|
||||||
|
if tc.state != "" {
|
||||||
|
id := "provision/n2/db"
|
||||||
|
if tc.refID != "" {
|
||||||
|
id = tc.refID
|
||||||
|
}
|
||||||
|
clear(provision)
|
||||||
|
key := "n1.m.db"
|
||||||
|
if tc.local != "" {
|
||||||
|
key += "." + tc.local
|
||||||
|
}
|
||||||
|
provision[key] = keyed(tc.state, tc.ref, id, tc.local)
|
||||||
|
}
|
||||||
|
vaultName = "provision/n2/db"
|
||||||
|
if tc.vault != "" {
|
||||||
|
vaultName = tc.vault
|
||||||
|
}
|
||||||
|
a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
|
||||||
|
got, _ := os.ReadFile(path)
|
||||||
|
if tc.want == "" {
|
||||||
|
if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, _ := os.Stat(path)
|
||||||
|
if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+29
-19
@@ -64,16 +64,16 @@ func foundDirectories(known store.State) []store.Applied {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// handOverLine is the operator's line that hands a directory to the mesh, with this node's name when it is
|
// handOverLine is the way out of a directory used as found, with this node's name when it is known: the node seat's
|
||||||
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
|
// hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5). The controller
|
||||||
// path is the resource's target beside the reason, never in it: the controller says a reason on the
|
// that answered `nox node hand-over` ended by its unassignment (issue 1126), so that line has no answerer. The path is
|
||||||
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
|
// the resource's target beside the reason, never in it: the operator's channel withholds a path (ADR 0234 §6).
|
||||||
func handOverLine() string {
|
func handOverLine() string {
|
||||||
node, _ := foundNode.Load().(string)
|
node, _ := foundNode.Load().(string)
|
||||||
if node == "" {
|
if node == "" {
|
||||||
node = "<node>"
|
node = "<node>"
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
|
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
|
||||||
}
|
}
|
||||||
|
|
||||||
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
||||||
@@ -129,16 +129,30 @@ func foundNow() []store.Applied {
|
|||||||
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
||||||
// apply, and a path that resolves elsewhere would give that elsewhere.
|
// apply, and a path that resolves elsewhere would give that elsewhere.
|
||||||
func handOverFound(statePath, path, by string) (string, error) {
|
func handOverFound(statePath, path, by string) (string, error) {
|
||||||
|
found, _, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
||||||
|
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at
|
||||||
|
// that path, as the engine states it, and what is on disk there.
|
||||||
|
func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) {
|
||||||
if !filepath.IsAbs(path) {
|
if !filepath.IsAbs(path) {
|
||||||
return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
||||||
}
|
}
|
||||||
if filepath.Clean(path) != path {
|
if filepath.Clean(path) != path {
|
||||||
return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
||||||
"separator); nothing was handed over", path)
|
"separator); nothing was handed over", path)
|
||||||
}
|
}
|
||||||
known, err := store.Load(statePath)
|
known, err := store.Load(statePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return store.Applied{}, nil, err
|
||||||
}
|
}
|
||||||
var found *store.Applied
|
var found *store.Applied
|
||||||
for _, r := range foundDirectories(known) {
|
for _, r := range foundDirectories(known) {
|
||||||
@@ -149,31 +163,27 @@ func handOverFound(statePath, path, by string) (string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if found == nil {
|
if found == nil {
|
||||||
return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
||||||
"the ones that are); nothing was handed over")
|
"the ones that are); nothing was handed over")
|
||||||
}
|
}
|
||||||
info, err := os.Lstat(path)
|
info, err := os.Lstat(path)
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
||||||
case info.Mode()&os.ModeSymlink != 0:
|
case info.Mode()&os.ModeSymlink != 0:
|
||||||
return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
||||||
case !info.IsDir():
|
case !info.IsDir():
|
||||||
return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
||||||
}
|
}
|
||||||
resolved, err := filepath.EvalSymlinks(path)
|
resolved, err := filepath.EvalSymlinks(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
||||||
}
|
}
|
||||||
if resolved != path {
|
if resolved != path {
|
||||||
return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
||||||
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
||||||
}
|
}
|
||||||
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
return *found, info, nil
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
|
||||||
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
|
// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
|
||||||
// every statement (novox/hq issue 339): the controller raises that as the module's condition. The way out it
|
// every statement (novox/hq issue 339): the controller raises that as the module's condition. The way out it
|
||||||
// names is the operator's line, with this node's name (novox/hq ADR 0272, issue 356).
|
// names is the node seat's hand-over, with this node's name (novox/hq ADR 0563 §5).
|
||||||
func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
known := store.State{Resources: []store.Applied{
|
known := store.State{Resources: []store.Applied{
|
||||||
@@ -35,14 +35,15 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
|||||||
t.Fatalf("stated as: %+v", r)
|
t.Fatalf("stated as: %+v", r)
|
||||||
}
|
}
|
||||||
// The path is the target, kept as evidence, and never in the reason: the operator's channel withholds a path
|
// The path is the target, kept as evidence, and never in the reason: the operator's channel withholds a path
|
||||||
// (novox/hq ADR 0234 §6), and the controller says the reason there.
|
// (novox/hq ADR 0234 §6). The way out is the node seat's hand-over (ADR 0563 §5), never the controller's terminal line,
|
||||||
if !strings.Contains(r.Reason, "`nox node hand-over laptop <directory>` on the control-node") ||
|
// which nothing answers since the controller ended (issue 1126).
|
||||||
|
if !strings.Contains(r.Reason, "`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") ||
|
||||||
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
||||||
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
|
t.Fatalf("the way out is not the node seat's hand-over, or carries the path: %q", r.Reason)
|
||||||
}
|
}
|
||||||
foundNode.Store("")
|
foundNode.Store("")
|
||||||
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||||
"`nox node hand-over <node> <directory>`") {
|
"`<node>/node-engine.hand-over`") {
|
||||||
t.Fatalf("before enrolment the line names a placeholder: %q", r.Reason)
|
t.Fatalf("before enrolment the line names a placeholder: %q", r.Reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,15 +3,15 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/liveness"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// owedBus is a link that takes reports and statements and keeps a have bucket in memory.
|
// owedBus is a link that takes reports and statements and keeps a have bucket in memory.
|
||||||
@@ -21,7 +21,6 @@ type owedBus struct {
|
|||||||
rv map[string]uint64
|
rv map[string]uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *owedBus) Report(context.Context, string, []byte) error { return nil }
|
|
||||||
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
||||||
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
||||||
|
|
||||||
@@ -45,10 +44,10 @@ func (b *owedBus) Set(_ context.Context, key string, value []byte, revision uint
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *owedBus) have(key string) (delivery.Have, bool) {
|
func (b *owedBus) have(key string) (nodeengine.HaveByNodeModuleValue, bool) {
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
defer b.mu.Unlock()
|
defer b.mu.Unlock()
|
||||||
h, err := delivery.DecodeHave(b.kv[key])
|
h, err := nodeengineclient.DecodeHave(b.kv[key])
|
||||||
return h, err == nil
|
return h, err == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,19 +66,19 @@ func TestAModuleWithNothingLongRunningIsOkWithinALookOfItsApply(t *testing.T) {
|
|||||||
}
|
}
|
||||||
fact := link.ModuleFact{Module: "xorg", FP: "src2:" + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("b", 64)}
|
fact := link.ModuleFact{Module: "xorg", FP: "src2:" + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("b", 64)}
|
||||||
applying := fact
|
applying := fact
|
||||||
applying.Phase = string(delivery.PhaseApplying)
|
applying.Phase = string(nodeengineclient.PhaseApplying)
|
||||||
queue.NoteModule(ctx, applying)
|
queue.NoteModule(ctx, applying)
|
||||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil)
|
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil, nil)
|
||||||
// The facts look inside the apply, while the module is still applying.
|
// The facts look inside the apply, while the module is still applying.
|
||||||
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
||||||
if h, _ := bus.have("n1.xorg"); h.Health != delivery.HealthUnknown {
|
if h, _ := bus.have("n1.xorg"); h.Health != nodeengineclient.HealthUnknown {
|
||||||
t.Fatalf("judged while applying: %+v", h)
|
t.Fatalf("judged while applying: %+v", h)
|
||||||
}
|
}
|
||||||
queue.NoteModule(ctx, fact)
|
queue.NoteModule(ctx, fact)
|
||||||
deadline := time.Now().Add(2 * time.Second) // well within one look (liveness.LookEvery, 15 s)
|
deadline := time.Now().Add(2 * time.Second) // well within one look (liveness.LookEvery, 15 s)
|
||||||
for {
|
for {
|
||||||
h, _ := bus.have("n1.xorg")
|
h, _ := bus.have("n1.xorg")
|
||||||
if h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
if h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if time.Now().After(deadline) {
|
if time.Now().After(deadline) {
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldBy is the containers an open maintenance window holds still, by runtime name (ADR 0189), at now.
|
||||||
|
func heldBy(windows *apply.Windows) func(time.Time) map[string]bool {
|
||||||
|
return func(now time.Time) map[string]bool {
|
||||||
|
held := map[string]bool{}
|
||||||
|
for _, w := range windows.Open(now) {
|
||||||
|
for _, name := range w.Holds {
|
||||||
|
held[name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inactiveIn is what the agent asks at its start of a module it believes runs (ADR 0568 §1): whether the machine shows a
|
||||||
|
// long-running resource the composition declares running not active, read from the service manager and the container
|
||||||
|
// runtime and from no record. A resource held by an open maintenance window, or waiting for a library, is not counted;
|
||||||
|
// one that cannot be read is not counted either (unknown is not inactive), and neither is one the manager is restarting.
|
||||||
|
func inactiveIn(rt liveness.Runtime, held func(time.Time) map[string]bool) func(context.Context, engcomp.Composed) bool {
|
||||||
|
return func(ctx context.Context, c engcomp.Composed) bool {
|
||||||
|
d, err := declarationOf(c)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var holding map[string]bool
|
||||||
|
if held != nil {
|
||||||
|
holding = held(time.Now())
|
||||||
|
}
|
||||||
|
return anyInactive(ctx, rt, liveness.LongRunning(d, withWaiting(nil, apply.WaitingOnLibrary(d))), holding)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// anyInactive reads each group of resources once (the containers, each service manager's units) and says whether any is
|
||||||
|
// not active.
|
||||||
|
func anyInactive(ctx context.Context, rt liveness.Runtime, resources []liveness.Resource, held map[string]bool) bool {
|
||||||
|
type group struct {
|
||||||
|
kind, scope, user string
|
||||||
|
targets []string
|
||||||
|
}
|
||||||
|
var groups []*group
|
||||||
|
at := map[string]*group{}
|
||||||
|
for _, r := range resources {
|
||||||
|
if r.Kind == liveness.KindContainer && held[r.Target] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g := &group{kind: r.Kind, scope: r.Scope, user: r.User}
|
||||||
|
if r.Kind == liveness.KindContainer {
|
||||||
|
g.scope, g.user = "", ""
|
||||||
|
}
|
||||||
|
key := g.kind + "/" + g.scope + "/" + g.user
|
||||||
|
if r.Kind != liveness.KindContainer {
|
||||||
|
key = "units/" + g.scope + "/" + g.user
|
||||||
|
}
|
||||||
|
if at[key] == nil {
|
||||||
|
at[key] = g
|
||||||
|
groups = append(groups, g)
|
||||||
|
}
|
||||||
|
at[key].targets = append(at[key].targets, r.Target)
|
||||||
|
}
|
||||||
|
for _, g := range groups {
|
||||||
|
var got map[string]liveness.Observed
|
||||||
|
var err error
|
||||||
|
if g.kind == liveness.KindContainer {
|
||||||
|
got, err = rt.Containers(ctx, g.targets)
|
||||||
|
} else {
|
||||||
|
got, err = rt.Units(ctx, g.scope, g.user, g.targets)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, t := range g.targets {
|
||||||
|
if o := got[t]; !o.Found || !(o.Running || o.Restarting) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1957/AC2 and AC4 (unit; ADR 0568 §2, §4): the judge's look acts on a resource that goes down, once per edge. A repeated
|
||||||
|
// statement acts nothing, the edge clears when the resource is healthy again, and a resource held, restarting, starting or
|
||||||
|
// missing (the other path's) is no edge.
|
||||||
|
func testOnlyAResourceNewlyDownIsAnEdge(t *testing.T) {
|
||||||
|
state := func(id, s, reason string, missing bool) liveness.State {
|
||||||
|
return liveness.State{Resource: liveness.Resource{Module: "dnsmasq", ID: id}, State: s, Reason: reason, Missing: missing}
|
||||||
|
}
|
||||||
|
down := state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, false)
|
||||||
|
steps := []struct {
|
||||||
|
name string
|
||||||
|
st []liveness.State
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"down: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||||
|
{"said again while down: none", []liveness.State{down}, nil},
|
||||||
|
{"healthy again: clears", []liveness.State{state("dnsmasq.unit", liveness.Healthy, "", false)}, nil},
|
||||||
|
{"down again: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||||
|
{"held by a window: none", []liveness.State{state("dnsmasq.unit", liveness.Held, "", false)}, nil},
|
||||||
|
{"restarting: none", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonRestarting, false)}, nil},
|
||||||
|
{"starting: none", []liveness.State{state("dnsmasq.unit", liveness.Starting, "", false)}, nil},
|
||||||
|
{"missing: the other path's", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, true)}, nil},
|
||||||
|
}
|
||||||
|
var last map[string]bool
|
||||||
|
for _, s := range steps {
|
||||||
|
var got []string
|
||||||
|
got, last = newlyInactive(last, liveness.Statement{Resources: s.st})
|
||||||
|
if len(got) != len(s.want) || (len(got) == 1 && got[0] != s.want[0]) {
|
||||||
|
t.Fatalf("%s: acted on %v, want %v", s.name, got, s.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeRuntime struct {
|
||||||
|
units map[string]liveness.Observed
|
||||||
|
containers map[string]liveness.Observed
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeRuntime) Containers(_ context.Context, _ []string) (map[string]liveness.Observed, error) {
|
||||||
|
return f.containers, f.err
|
||||||
|
}
|
||||||
|
func (f fakeRuntime) Units(_ context.Context, _, _ string, _ []string) (map[string]liveness.Observed, error) {
|
||||||
|
return f.units, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1957/AC1 (unit; ADR 0568 §1): the machine's facts say a declared-running resource is not active when its unit is stopped
|
||||||
|
// or unknown to the manager; everything up, a restart in progress, a read that failed and a container held by a
|
||||||
|
// maintenance window are not.
|
||||||
|
func testTheMachineShowsAResourceNotActive(t *testing.T) {
|
||||||
|
svc := []byte(`{"id":"dnsmasq.unit","type":"service","unit":"dnsmasq.service","state":"running","module":"dnsmasq"}`)
|
||||||
|
ctr := []byte(`{"id":"dnsmasq.box","type":"container","name":"box","image":"x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","module":"dnsmasq"}`)
|
||||||
|
up := liveness.Observed{Found: true, Running: true}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
rt fakeRuntime
|
||||||
|
held map[string]bool
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"all up", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||||
|
{"unit inactive", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||||
|
{"unit unknown to the manager", fakeRuntime{containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||||
|
{"unit restarting", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true, Restarting: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||||
|
{"container not running", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, nil, true},
|
||||||
|
{"container held by a window", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, map[string]bool{"box": true}, false},
|
||||||
|
{"unread", fakeRuntime{err: errors.New("the service manager could not be read")}, nil, false},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
d, err := declarationOf(engcomp.Composed{Node: "n1", Module: "dnsmasq", Resources: []json.RawMessage{svc, ctr}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := anyInactive(context.Background(), c.rt, liveness.LongRunning(d, nil), c.held); got != c.want {
|
||||||
|
t.Fatalf("inactive = %v, want %v", got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the
|
||||||
|
// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine
|
||||||
|
// reads them and checks them against the digest. Never another module's files: the seat's verb is the way.
|
||||||
|
|
||||||
|
// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object.
|
||||||
|
var (
|
||||||
|
keptAskWithin = 30 * time.Second
|
||||||
|
keptObjectWithin = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build
|
||||||
|
// holds the seat of the language it was built in, and the others do not answer.
|
||||||
|
var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat}
|
||||||
|
|
||||||
|
// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node.
|
||||||
|
func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error),
|
||||||
|
ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error),
|
||||||
|
object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch {
|
||||||
|
return func(ctx context.Context, module, digest string) ([]byte, bool, error) {
|
||||||
|
node, fp, kept, err := buildOf(ctx, module, digest)
|
||||||
|
if err != nil || !kept {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
hex := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
var notHeld []string
|
||||||
|
for _, seat := range keptBuildSeats {
|
||||||
|
raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact,
|
||||||
|
nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin)
|
||||||
|
if err != nil {
|
||||||
|
notHeld = append(notHeld, seat+": "+err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var got nodebuildgo.KeptArtifactAnswer
|
||||||
|
if json.Unmarshal(raw, &got) != nil {
|
||||||
|
return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case got.Error != "":
|
||||||
|
return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error)
|
||||||
|
case got.Refused != "":
|
||||||
|
return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words)
|
||||||
|
case got.Store == "" || got.Object == "" || got.Digest != hex:
|
||||||
|
return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex)
|
||||||
|
}
|
||||||
|
octx, cancel := context.WithTimeout(ctx, keptObjectWithin)
|
||||||
|
body, err := object(octx, got.Store, got.Object, got.Size+1)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err)
|
||||||
|
}
|
||||||
|
return body, true, nil
|
||||||
|
}
|
||||||
|
return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)",
|
||||||
|
node, module, strings.Join(notHeld, "; "))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47).
|
||||||
|
func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) {
|
||||||
|
hex := strings.Repeat("a", 64)
|
||||||
|
stored := func(string) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
kept bool
|
||||||
|
answer func(seat string) (json.RawMessage, error)
|
||||||
|
object error
|
||||||
|
wantKept bool
|
||||||
|
wantErr string
|
||||||
|
wantSeat string
|
||||||
|
}{
|
||||||
|
{"a build kept nothing is not asked", false, nil, nil, false, "", ""},
|
||||||
|
{"the seat of the language answers", true, func(seat string) (json.RawMessage, error) {
|
||||||
|
if seat != "node-build-node" {
|
||||||
|
return nil, errors.New("no responder")
|
||||||
|
}
|
||||||
|
return stored(seat)
|
||||||
|
}, nil, true, "", "node-build-node"},
|
||||||
|
{"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") },
|
||||||
|
nil, true, "novox keeps the build of distribution", ""},
|
||||||
|
{"the seat refuses it", true, func(string) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil
|
||||||
|
}, nil, true, "refuses the kept build of distribution: not kept", ""},
|
||||||
|
{"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
var seats []string
|
||||||
|
fetch := keptArtifactFetch(
|
||||||
|
func(context.Context, string, string) (string, string, bool, error) {
|
||||||
|
return "novox", "f1", row.kept, nil
|
||||||
|
},
|
||||||
|
func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) {
|
||||||
|
if node != "novox" || verb != "kept-artifact" {
|
||||||
|
t.Errorf("asked %s of %s on %s", verb, seat, node)
|
||||||
|
}
|
||||||
|
seats = append(seats, seat)
|
||||||
|
return row.answer(seat)
|
||||||
|
},
|
||||||
|
func(_ context.Context, store, name string, limit int64) ([]byte, error) {
|
||||||
|
if store != "kept-builds" || name != hex || limit < 5 {
|
||||||
|
t.Errorf("read %s/%s up to %d", store, name, limit)
|
||||||
|
}
|
||||||
|
return []byte("bytes"), row.object
|
||||||
|
})
|
||||||
|
body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex)
|
||||||
|
if kept != row.wantKept {
|
||||||
|
t.Fatalf("kept = %v", kept)
|
||||||
|
}
|
||||||
|
if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) {
|
||||||
|
t.Fatalf("got %q, %v", body, err)
|
||||||
|
}
|
||||||
|
if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) {
|
||||||
|
t.Fatalf("the error is %v, want %q", err, row.wantErr)
|
||||||
|
}
|
||||||
|
if !row.kept && len(seats) > 0 {
|
||||||
|
t.Errorf("a build that kept nothing asked %v", seats)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||||
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
@@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
|||||||
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||||
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
|
||||||
|
// open, nor after it ended; and a library waiting for its disk is not asked.
|
||||||
|
t.Run("the home engine asks the take once", func(t *testing.T) {
|
||||||
|
var asked []oc.Ask
|
||||||
|
state := &link.MemState{}
|
||||||
|
h := &link.HandActs{Node: "n1", State: state,
|
||||||
|
JudgeAccess: func(act, access string) (link.Judged, error) {
|
||||||
|
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
|
||||||
|
},
|
||||||
|
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
|
||||||
|
h.Asker = &occ.Client{Module: occ.EngineAsker("n1"), State: state,
|
||||||
|
Publish: func(subject string, body any, id string) error {
|
||||||
|
if a, ok := body.(oc.Ask); ok {
|
||||||
|
asked = append(asked, a)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
waiting := keepWaiting(vs)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
|
||||||
|
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
|
||||||
|
}
|
||||||
|
h.Heard(oc.Warrant{Ask: asked[0].ID, Asker: occ.EngineAsker("n1"), Outcome: occ.OutcomeExpired, At: occ.FormatTime(time.Now())})
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
|
||||||
|
}
|
||||||
|
// Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again.
|
||||||
|
keys, _ := state.Keys("act.")
|
||||||
|
for _, k := range keys {
|
||||||
|
_ = state.Delete(k)
|
||||||
|
}
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again with the acts gone: %d", len(asked))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// loginsFile is where this engine keeps the logins it holds between runs (novox/hq ADR 0583 §1): beside its identity, readable by
|
||||||
|
// its owner alone, as the identity's membership password is.
|
||||||
|
func loginsFile(state string) string {
|
||||||
|
return filepath.Join(filepath.Dir(identity.Path(state)), "logins.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadLogins is the engine's logins: what it kept, with the membership's password as the first login's.
|
||||||
|
func loadLogins(state, node, first string) *link.Logins {
|
||||||
|
path := loginsFile(state)
|
||||||
|
var saved link.LoginsState
|
||||||
|
if raw, err := os.ReadFile(path); err == nil {
|
||||||
|
_ = json.Unmarshal(raw, &saved)
|
||||||
|
}
|
||||||
|
return link.NewLogins(node, first, saved, func(s link.LoginsState) error { return writeLogins(path, s) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLogins(path string, s link.LoginsState) error {
|
||||||
|
raw, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp := path + ".new"
|
||||||
|
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loginsFollowAgain bounds the wait before a watch of node-logins that could not be made is tried again on the same link:
|
||||||
|
// the vault may not serve rotation yet, and until it does this node is not granted the read.
|
||||||
|
var loginsFollowAgain = 2 * time.Minute
|
||||||
|
|
||||||
|
// followLogins follows this node's key of the vault's node-logins on every link, as a watch (never a poll), for as long as ctx
|
||||||
|
// lasts, and hands each value to hear. A watch that cannot be made is said once and tried again.
|
||||||
|
func followLogins(ctx context.Context, session *link.Session, node string, hear func(meshvault.NodeLoginsByNodeValue), say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
watching, stop := context.WithCancel(linked)
|
||||||
|
updates, werr := session.KVWatch(watching, buckets.NodeLogins, node)
|
||||||
|
if werr != nil {
|
||||||
|
stop()
|
||||||
|
if line := "this node's login is not followed yet (" + werr.Error() + "): it dials as it last did, and looks again in " + loginsFollowAgain.String(); line != said {
|
||||||
|
said = line
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
said = ""
|
||||||
|
for u := range updates {
|
||||||
|
if u.Caught || u.Deleted || len(u.Value) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nl, err := meshvaultclient.DecodeNodeLogin(u.Value)
|
||||||
|
if err != nil {
|
||||||
|
say("the key " + node + " of " + buckets.NodeLoginsRead + " is not read: " + err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hear(nl)
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
case <-time.After(loginsFollowAgain):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchLogin asks the vault, through its existing hand-out of a machine's own secrets, for the value of login, sealed to this
|
||||||
|
// node's key, and opens it with the sealing key; no new verb is asked for it.
|
||||||
|
func fetchLogin(queue *link.Session, statePath string) func(context.Context, string) (string, error) {
|
||||||
|
unseal := sealOpener(statePath)
|
||||||
|
return func(ctx context.Context, login string) (string, error) {
|
||||||
|
// The vault names the login values under the engine's own module, which is §527's (§634).
|
||||||
|
own, err := queue.Own.Wait(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the node-engine's own module is not named yet, so the vault's name of its login value is not known: %w", err)
|
||||||
|
}
|
||||||
|
raw, err := queue.AskVault(ctx, "own", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Own []struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
|
} `json:"own"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return "", fmt.Errorf("the vault's own answer is not one: %w", err)
|
||||||
|
}
|
||||||
|
for _, o := range a.Own {
|
||||||
|
if o.Module != own || o.Name != login {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sealed := string(o.Sealed)
|
||||||
|
var text string
|
||||||
|
if json.Unmarshal(o.Sealed, &text) == nil {
|
||||||
|
sealed = text
|
||||||
|
}
|
||||||
|
value, err := unseal(sealed)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the value of %s cannot be opened with this node's sealing key: %w", login, err)
|
||||||
|
}
|
||||||
|
return string(value), nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), own, login)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func queueNode(q *link.Session) string { return q.Membership.Node }
|
||||||
|
|
||||||
|
// newRotator is the engine's side of a rotation over this session (ADR 0583 §3-§6): the value fetched through the vault's own
|
||||||
|
// hand-out, the test-dial on a second connection, the confirmation over the new link, and a fall back carried in the health
|
||||||
|
// report.
|
||||||
|
func newRotator(queue *link.Session, statePath string, timeout time.Duration, say link.Announce) *link.Rotator {
|
||||||
|
r := &link.Rotator{Logins: queue.Membership.Logins, Say: say,
|
||||||
|
Fetch: fetchLogin(queue, statePath),
|
||||||
|
TestDial: func(ctx context.Context, login, password string) error {
|
||||||
|
return link.TestDial(ctx, queue.Membership, login, password, timeout)
|
||||||
|
},
|
||||||
|
Confirm: func(ctx context.Context) error {
|
||||||
|
login, _ := queue.Membership.Logins.Current()
|
||||||
|
if err := queue.WhenLinkedAs(ctx, login); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := queue.AskVault(ctx, "node-account-confirm", map[string]any{})
|
||||||
|
return err
|
||||||
|
}}
|
||||||
|
r.Watch()
|
||||||
|
// The fall back is carried in this node's health report, from which the watchman raises node-account-fallback (ADR 0583 §6).
|
||||||
|
queue.Fallback = r.Fallback
|
||||||
|
return r
|
||||||
|
}
|
||||||
+280
-185
@@ -16,6 +16,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
@@ -34,6 +36,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/agent"
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/bundle"
|
"github.com/novox/mesh-host/internal/bundle"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
"github.com/novox/mesh-host/internal/firewall"
|
"github.com/novox/mesh-host/internal/firewall"
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
"github.com/novox/mesh-host/internal/inventory"
|
"github.com/novox/mesh-host/internal/inventory"
|
||||||
@@ -41,6 +44,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/liveness"
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
"github.com/novox/mesh-host/internal/network"
|
"github.com/novox/mesh-host/internal/network"
|
||||||
"github.com/novox/mesh-host/internal/outward"
|
"github.com/novox/mesh-host/internal/outward"
|
||||||
|
"github.com/novox/mesh-host/internal/processbuilds"
|
||||||
"github.com/novox/mesh-host/internal/profile"
|
"github.com/novox/mesh-host/internal/profile"
|
||||||
"github.com/novox/mesh-host/internal/reachable"
|
"github.com/novox/mesh-host/internal/reachable"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
@@ -99,7 +103,7 @@ func versionAt(self, stamped string) string {
|
|||||||
const usage = `nox-mesh-host — the node-engine
|
const usage = `nox-mesh-host — the node-engine
|
||||||
|
|
||||||
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
|
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
|
||||||
operator's command is nox (mesh-cli), on any node, from their own account (novox/hq ADR 0272);
|
operator's command is nox (mesh-cli), on any node, from their own account, through the seats' verbs;
|
||||||
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
|
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
|
||||||
reach, run as root at the machine.
|
reach, run as root at the machine.
|
||||||
|
|
||||||
@@ -109,8 +113,7 @@ reach, run as root at the machine.
|
|||||||
key make this machine's tunnel key, or read the one it made, and print the public
|
key make this machine's tunnel key, or read the one it made, and print the public
|
||||||
half: what its join token is issued for (novox/hq ADR 0169)
|
half: what its join token is issued for (novox/hq ADR 0169)
|
||||||
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
||||||
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
overlay take refused: the controller that took a rekey ended and no seat takes one now
|
||||||
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
|
||||||
owned what this host has applied and still owns
|
owned what this host has applied and still owns
|
||||||
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
||||||
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
||||||
@@ -156,6 +159,17 @@ type options struct {
|
|||||||
file string
|
file string
|
||||||
// out is where what a command says goes. Stdout, and a buffer under test.
|
// out is where what a command says goes. Stdout, and a buffer under test.
|
||||||
out io.Writer
|
out io.Writer
|
||||||
|
// runner is what the node agent's applies run commands through (issue 774, ADR 0594): nil is the machine's own,
|
||||||
|
// apply.ExecRunner; a test gives one that records what an apply asked of the container runtime.
|
||||||
|
runner apply.Runner
|
||||||
|
}
|
||||||
|
|
||||||
|
// run is the runner the node agent's applies use: the options' own, else the machine's.
|
||||||
|
func (o options) run() apply.Runner {
|
||||||
|
if o.runner != nil {
|
||||||
|
return o.runner
|
||||||
|
}
|
||||||
|
return apply.ExecRunner
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseArgs takes the subcommand first, then its flags.
|
// parseArgs takes the subcommand first, then its flags.
|
||||||
@@ -516,7 +530,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
|||||||
|
|
||||||
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
||||||
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
||||||
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||||
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -808,162 +822,21 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
|
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
|
||||||
reported := profileAsReported(detected)
|
reported := profileAsReported(detected)
|
||||||
|
|
||||||
// Signed with the identity just generated, so the mesh can tell this machine from anyone else
|
// Through the seat mesh-node-enroller (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended
|
||||||
// who knows its public key (novox/hq issue 083).
|
// with the controller (issue 1126).
|
||||||
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
|
return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath)
|
||||||
sealing.Public, serving.Public))
|
|
||||||
// The token says where to go and which certificate that address must present. It says nothing
|
|
||||||
// about which bus is there, and does not need to: there is one, and this host knows which
|
|
||||||
// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
|
|
||||||
//
|
|
||||||
// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
|
|
||||||
// while two buses existed and became a refusal the moment one did: an empty transport is not
|
|
||||||
// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
|
|
||||||
// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
|
|
||||||
// changed.
|
|
||||||
reply, err := link.Enrol(ctx,
|
|
||||||
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
|
|
||||||
*name, token.Secret,
|
|
||||||
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
|
|
||||||
opts.timeout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The mesh's name for this node wins over what the machine called itself: the token was
|
|
||||||
// issued for a node record, and that record is what the identity binds to.
|
|
||||||
mine.Node = reply.Node
|
|
||||||
mine.Membership = identity.Membership{
|
|
||||||
Broker: firstNonEmpty(reply.Broker, token.Broker),
|
|
||||||
Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
|
|
||||||
Signer: firstNonEmpty2(reply.Signer, token.Signer),
|
|
||||||
Password: reply.Password,
|
|
||||||
// **Which bus this membership is for, said rather than left empty** (novox/hq
|
|
||||||
// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
|
|
||||||
// is not this one's — so a node enrolled without it came up and reconnected for ever
|
|
||||||
// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
|
|
||||||
// reply does not carry it because there is one bus and the host knows which (ADR 0131);
|
|
||||||
// what was missing was writing that down where the link reads it.
|
|
||||||
Transport: link.OnNATS,
|
|
||||||
}
|
|
||||||
if mine.Membership.Password == "" {
|
|
||||||
// The mesh did not replace the token's secret, so it is still this node's broker
|
|
||||||
// password. Said rather than silently kept: a one-time secret living on as a credential
|
|
||||||
// is worth knowing about.
|
|
||||||
mine.Membership.Password = token.Secret
|
|
||||||
fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " +
|
|
||||||
"remains this node's credential")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Saved only now, and only once the mesh has said it knows this node. A node holding an
|
|
||||||
// identity the mesh has never recorded would believe it had joined and be believed by
|
|
||||||
// nobody — worse than not having joined, because nothing would look wrong.
|
|
||||||
if err := identity.Save(identityPath, mine); err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"the mesh accepted this node as %q and its identity could not be saved: %w\n"+
|
|
||||||
"That token is spent, so getting back needs a new one", reply.Node, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !mine.Membership.Joined() {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"the mesh accepted this node as %q but did not say how to reach it again, so this "+
|
|
||||||
"identity could not be used after a restart. Nothing was saved", reply.Node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Written before the identity, so a node that dies between the two has a key file with no
|
|
||||||
// identity — which enrols again cleanly — rather than an identity naming a key that is not
|
|
||||||
// there, which looks joined and cannot come up.
|
|
||||||
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
||||||
[]byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's overlay key: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(identity.SealingKeyPath(opts.state),
|
|
||||||
[]byte(sealing.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's sealing key: %w", err)
|
|
||||||
}
|
|
||||||
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's serving key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("\nenrolled as %s\n", reply.Node)
|
|
||||||
fmt.Printf(" identity %s\n", identityPath)
|
|
||||||
fmt.Printf(" queue %s\n", reply.Queue)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
|
// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled
|
||||||
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
|
// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which
|
||||||
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
|
// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows
|
||||||
// every credential the mesh sealed to it.
|
// would raise the mesh's interface on a key its hub does not peer.
|
||||||
func overlayCommand(ctx context.Context, opts options) error {
|
func overlayCommand(_ context.Context, opts options) error {
|
||||||
if opts.file != "take" {
|
if opts.file != "take" {
|
||||||
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
||||||
}
|
}
|
||||||
identityPath := identity.Path(opts.state)
|
return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " +
|
||||||
mine, err := identity.Load(identityPath)
|
"takes one now. Nothing was changed")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w. Nothing was changed", err)
|
|
||||||
}
|
|
||||||
taken, rekey, err := rekeyOnto(mine, found)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
|
|
||||||
fmt.Printf(" identity, sealing and serving keys are untouched\n")
|
|
||||||
|
|
||||||
// Told first, then written: a mesh told and a machine not yet written is put right by running
|
|
||||||
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
|
|
||||||
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
|
|
||||||
// on a key the mesh does not know at the next restart.
|
|
||||||
if err := link.Publish(ctx, link.Membership{
|
|
||||||
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
|
|
||||||
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
|
|
||||||
Transport: mine.Membership.Transport,
|
|
||||||
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
|
|
||||||
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
|
|
||||||
|
|
||||||
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
||||||
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
|
|
||||||
}
|
|
||||||
if err := identity.Save(identityPath, taken); err != nil {
|
|
||||||
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
|
|
||||||
identity.OverlayKeyPath(opts.state))
|
|
||||||
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
|
|
||||||
mine.Node, found.Port, mine.Node)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
|
|
||||||
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
|
|
||||||
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
|
|
||||||
// is the one before the take, so the mesh can tell a repeat from a replay.
|
|
||||||
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
|
|
||||||
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
|
|
||||||
if err != nil {
|
|
||||||
return identity.Identity{}, link.Rekey{}, err
|
|
||||||
}
|
|
||||||
previous := mine.Overlay.Public
|
|
||||||
if previous == overlay.Public && mine.OverlayBefore != "" {
|
|
||||||
previous = mine.OverlayBefore
|
|
||||||
}
|
|
||||||
taken := mine
|
|
||||||
taken.Overlay = overlay
|
|
||||||
if previous != overlay.Public {
|
|
||||||
taken.OverlayBefore = previous
|
|
||||||
}
|
|
||||||
presented := carried(found)
|
|
||||||
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
|
|
||||||
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
|
|
||||||
return taken, rekey, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
||||||
@@ -1046,15 +919,7 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
say(err.Error())
|
say(err.Error())
|
||||||
}
|
}
|
||||||
windows := apply.WindowsIn(filepath.Dir(opts.state))
|
windows := apply.WindowsIn(filepath.Dir(opts.state))
|
||||||
j.HeldNow = func(now time.Time) map[string]bool {
|
j.HeldNow = heldBy(windows)
|
||||||
held := map[string]bool{}
|
|
||||||
for _, w := range windows.Open(now) {
|
|
||||||
for _, name := range w.Holds {
|
|
||||||
held[name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return held
|
|
||||||
}
|
|
||||||
judging = j
|
judging = j
|
||||||
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
||||||
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
||||||
@@ -1074,6 +939,11 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
// which the launcher already reads as "run whatever is on disk now".
|
// which the launcher already reads as "run whatever is on disk now".
|
||||||
aside, standAside := context.WithCancel(ctx)
|
aside, standAside := context.WithCancel(ctx)
|
||||||
defer standAside()
|
defer standAside()
|
||||||
|
// **And it ends between applies too** (novox/hq issue 1585): whichever way the link ends, the apply in flight ends and
|
||||||
|
// saves before the process does. Deferred after standAside, so it runs before it: nothing is cancelled under it.
|
||||||
|
defer func() { _ = stopBetweenApplies(say, stopBound) }()
|
||||||
|
// What an earlier stop cut off before its rename is removed, and said, before anything saves.
|
||||||
|
removeLeftoverSaves(opts.state, say)
|
||||||
stoodAside := false
|
stoodAside := false
|
||||||
membership := link.Membership{
|
membership := link.Membership{
|
||||||
Node: mine.Node,
|
Node: mine.Node,
|
||||||
@@ -1082,6 +952,8 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
Password: mine.Membership.Password,
|
Password: mine.Membership.Password,
|
||||||
Transport: mine.Membership.Transport,
|
Transport: mine.Membership.Transport,
|
||||||
Signer: mine.Membership.Signer,
|
Signer: mine.Membership.Signer,
|
||||||
|
// Which of its two logins it dials as (ADR 0583 §1): the one the vault's node-logins names.
|
||||||
|
Logins: loadLogins(opts.state, mine.Node, mine.Membership.Password),
|
||||||
}
|
}
|
||||||
|
|
||||||
// What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus
|
// What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus
|
||||||
@@ -1113,8 +985,23 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
||||||
// controller asks.
|
// controller asks.
|
||||||
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
||||||
|
// What this engine trusts a mesh signature from: the vault's key alone, once read from its signers (novox/hq ADR 0550
|
||||||
|
// §2; issue 1126: the controller's key never, the controller ended by its unassignment).
|
||||||
|
trust := newMeshTrust()
|
||||||
|
own := link.NewOwnName()
|
||||||
|
// The engine's own module, first from its applied record of the build whose fingerprint is its own (design 47 G §527,
|
||||||
|
// §634, A-149); on a first start with no such record, from its first composing.
|
||||||
|
own.Set(ownModuleApplied(appliedStoreFor(opts.state), os.Getenv(compositions.FingerprintVar)))
|
||||||
|
// The live start's recovery (A-404): a first start whose composing never ends is named by its unit's environment, so it
|
||||||
|
// serves its seat (root-free); the name the assignments place on this node. Ended by the lookup from the bus after G.
|
||||||
|
if own.Get() == "" {
|
||||||
|
own.Set(os.Getenv("MESH_ENGINE_OWN_MODULE"))
|
||||||
|
}
|
||||||
queue := &link.Session{
|
queue := &link.Session{
|
||||||
|
Own: own,
|
||||||
Membership: membership,
|
Membership: membership,
|
||||||
|
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||||
|
Keys: recordKeys(opts.state, trust.Trusts),
|
||||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||||
HaveWritten: proof.haveWritten,
|
HaveWritten: proof.haveWritten,
|
||||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||||
@@ -1135,7 +1022,29 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
Say: say,
|
Say: say,
|
||||||
Timeout: opts.timeout,
|
Timeout: opts.timeout,
|
||||||
}
|
}
|
||||||
|
// And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the
|
||||||
|
// operator under node-engine-<node>, and its acts kept beside its state.
|
||||||
|
queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch)
|
||||||
|
go reconcileActs(ctx, queue.Acts)
|
||||||
current := &agentNow{}
|
current := &agentNow{}
|
||||||
|
queue.Acts.Composition = compositionFrom(current, queue.WithheldOf)
|
||||||
|
queue.Acts.Compose = composeFrom(current, &composerNow)
|
||||||
|
queue.Acts.Packages = packagesFrom(mine.Node, queue, &composerNow)
|
||||||
|
queue.Acts.FirstViews, queue.Acts.EntryAgain = firstViewsFrom(&composerNow), entryAgainFrom(&composerNow)
|
||||||
|
queue.Acts.SeatList, queue.Acts.SeatListAgain = seatListRead, seatListAgain
|
||||||
|
queue.Acts.ProcessBuilds = func(module string) (any, error) {
|
||||||
|
d, err := declaredNow(opts.state, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
d, err = nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nodeengine.ProcessBuilds{Node: mine.Node, Error: "what runs on this node does not read: " + err.Error()}, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
|
||||||
|
defer cancel()
|
||||||
|
return processbuilds.Reader{Node: mine.Node, Run: processbuilds.Runner(opts.run())}.Read(ctx, d, module), nil
|
||||||
|
}
|
||||||
|
queue.Acts.ComposeCandidate = composeCandidateFrom(mine.Node, &shadowNow)
|
||||||
setRunsHere(runsFromAgent(current))
|
setRunsHere(runsFromAgent(current))
|
||||||
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
||||||
sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
|
sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
|
||||||
@@ -1171,11 +1080,34 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
if ag := current.get(); ag != nil {
|
if ag := current.get(); ag != nil {
|
||||||
ag.Retry(modules...)
|
ag.Retry(modules...)
|
||||||
}
|
}
|
||||||
|
}, func(modules ...string) {
|
||||||
|
if ag := current.get(); ag != nil {
|
||||||
|
ag.Reconcile(modules...)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
||||||
// busy containers are asked for again soon (ADR 0360 §11).
|
// busy containers are asked for again soon (ADR 0360 §11).
|
||||||
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
|
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
|
||||||
|
trust.onChange(func() { factsOfNow(aside, current.get()) })
|
||||||
|
go followTrust(aside, queue, trust, say)
|
||||||
|
// And the status view's `settled`, for a failed pull from a store to be tried again once the store's holder has settled
|
||||||
|
// (design 51, ENG-11).
|
||||||
|
go followSettled(aside, queue, func() settledTold {
|
||||||
|
if ag := current.get(); ag != nil {
|
||||||
|
return ag
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}, say)
|
||||||
|
// And the login it dials as, which the vault's node-logins names for this node (novox/hq ADR 0583 §1).
|
||||||
|
rotator := newRotator(queue, opts.state, opts.timeout, say)
|
||||||
|
go followLogins(aside, queue, mine.Node, func(nl meshvault.NodeLoginsByNodeValue) {
|
||||||
|
if line := membership.Logins.Heard(nl); line != "" {
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
rotator.Heard(aside, nl)
|
||||||
|
}, say)
|
||||||
|
go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow)
|
||||||
go busyPasses(aside, func() {
|
go busyPasses(aside, func() {
|
||||||
if ag := current.get(); ag != nil {
|
if ag := current.get(); ag != nil {
|
||||||
ag.Retry(busyModules()...)
|
ag.Retry(busyModules()...)
|
||||||
@@ -1183,12 +1115,9 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
}, BusyAskEvery)
|
}, BusyAskEvery)
|
||||||
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
|
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
|
||||||
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
|
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
|
||||||
// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
|
|
||||||
// (novox/hq ADR 0272).
|
|
||||||
go serveMeshCLI(aside, queue, say)
|
|
||||||
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
||||||
// 0349 §4).
|
// 0349 §4).
|
||||||
go serveFilling(aside, opts, mine.Membership.Signer, say)
|
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
|
||||||
|
|
||||||
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
||||||
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
||||||
@@ -1206,8 +1135,6 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
// be a control surface on every machine, reachable by anything that can reach the machine, in
|
// be a control surface on every machine, reachable by anything that can reach the machine, in
|
||||||
// exchange for saving twenty seconds — and the whole security argument rests on there not being
|
// exchange for saving twenty seconds — and the whole security argument rests on there not being
|
||||||
// one. A signal is delivered by the service manager to a process it already supervises.
|
// one. A signal is delivered by the service manager to a process it already supervises.
|
||||||
// (mesh-cli's socket is no such surface: it has no network address and nothing sent to it acts on
|
|
||||||
// the machine — it is asked of the controller, novox/hq ADR 0272 §3.)
|
|
||||||
//
|
//
|
||||||
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
|
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
|
||||||
// and nothing here is being reloaded from a file that a different signal would suit better.
|
// and nothing here is being reloaded from a file that a different signal would suit better.
|
||||||
@@ -1334,13 +1261,16 @@ const (
|
|||||||
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
||||||
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
||||||
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
||||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
|
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply, reconcile func(modules ...string)) {
|
||||||
var missingBefore map[string]bool
|
say("the looks of what runs and of the accounts start (A-405)")
|
||||||
|
var missingBefore, inactiveBefore map[string]bool
|
||||||
ticker := time.NewTicker(liveness.LookEvery)
|
ticker := time.NewTicker(liveness.LookEvery)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
var lastSaid time.Time
|
var lastSaid time.Time
|
||||||
owed := true
|
owed := true
|
||||||
spaced := 1.0
|
spaced := 1.0
|
||||||
|
// healthUnwritten is why the health report was last not written, said once each time it changes.
|
||||||
|
var healthUnwritten string
|
||||||
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
||||||
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
||||||
var nudged <-chan struct{}
|
var nudged <-chan struct{}
|
||||||
@@ -1355,8 +1285,16 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
case <-nudged:
|
case <-nudged:
|
||||||
owed = true
|
owed = true
|
||||||
}
|
}
|
||||||
|
first := lastAccounts.Load() == nil
|
||||||
|
began := time.Now()
|
||||||
|
if first {
|
||||||
|
say("the first look of what runs begins (A-405)")
|
||||||
|
}
|
||||||
st, changed := j.Look(ctx)
|
st, changed := j.Look(ctx)
|
||||||
owed = owed || changed
|
owed = owed || changed
|
||||||
|
if first {
|
||||||
|
say(fmt.Sprintf("the first look of what runs took %s; units and accounts next (A-405)", time.Since(began).Round(time.Millisecond)))
|
||||||
|
}
|
||||||
var unitSt *units.Statement
|
var unitSt *units.Statement
|
||||||
if u := unitJudge; u != nil {
|
if u := unitJudge; u != nil {
|
||||||
us, unitsChanged := u.Look(ctx)
|
us, unitsChanged := u.Look(ctx)
|
||||||
@@ -1370,8 +1308,15 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
}
|
}
|
||||||
var accountSt *accounts.Statement
|
var accountSt *accounts.Statement
|
||||||
if a := accountJudge; a != nil {
|
if a := accountJudge; a != nil {
|
||||||
|
if first {
|
||||||
|
say(fmt.Sprintf("the first look of the accounts begins at %s (A-405)", time.Since(began).Round(time.Millisecond)))
|
||||||
|
}
|
||||||
as, accountsChanged := a.Look(ctx)
|
as, accountsChanged := a.Look(ctx)
|
||||||
accountSt = &as
|
accountSt = &as
|
||||||
|
lastAccounts.Store(&as)
|
||||||
|
if first {
|
||||||
|
say(fmt.Sprintf("the first look of the accounts ended at %s: %d accounts (A-405)", time.Since(began).Round(time.Millisecond), len(as.Accounts)))
|
||||||
|
}
|
||||||
owed = owed || accountsChanged
|
owed = owed || accountsChanged
|
||||||
if accountsChanged {
|
if accountsChanged {
|
||||||
for _, v := range as.Accounts {
|
for _, v := range as.Accounts {
|
||||||
@@ -1386,6 +1331,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
accessSt, accessesChanged := accesses.look(time.Now())
|
accessSt, accessesChanged := accesses.look(time.Now())
|
||||||
keeping.poke(ctx)
|
keeping.poke(ctx)
|
||||||
owed = owed || accessesChanged
|
owed = owed || accessesChanged
|
||||||
|
// And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq
|
||||||
|
// ADR 0563 §4).
|
||||||
|
askTakes(queue.Acts, keepWaiting(accessSt), say)
|
||||||
if accessesChanged {
|
if accessesChanged {
|
||||||
for _, v := range accessSt {
|
for _, v := range accessSt {
|
||||||
if v.State != link.StateHealthy {
|
if v.State != link.StateHealthy {
|
||||||
@@ -1420,6 +1368,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
||||||
accessesHealthy(accessSt)
|
accessesHealthy(accessSt)
|
||||||
|
h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||||
|
unitSt), accountSt), found), accessSt)
|
||||||
|
// The node's own health report in its key on the bus, compared on every look and written only when what it says
|
||||||
|
// changed (issue 774): what the watchman raises the machine's alerts from, without the controller.
|
||||||
|
writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say)
|
||||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -1437,14 +1390,57 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
|
say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
|
||||||
reapply(m...)
|
reapply(m...)
|
||||||
}
|
}
|
||||||
|
// A resource its module declares running that is down but still known to the manager is started by the same
|
||||||
|
// apply, once each time it goes down, never on a clock (ADR 0568 §2): the judge reads, the agent acts.
|
||||||
|
var inactive []string
|
||||||
|
inactive, inactiveBefore = newlyInactive(inactiveBefore, st)
|
||||||
|
if len(inactive) > 0 && reconcile != nil {
|
||||||
|
say(fmt.Sprintf("applying %s again: a resource it declares running is not active", strings.Join(inactive, ", ")))
|
||||||
|
reconcile(inactive...)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
if queue.SayHealth(ctx, h) {
|
||||||
unitSt), accountSt), found), accessSt)) {
|
|
||||||
lastSaid, owed = time.Now(), false
|
lastSaid, owed = time.Now(), false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could
|
||||||
|
// not is said once per reason, and said again when it is written after all.
|
||||||
|
func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string,
|
||||||
|
say link.Announce) {
|
||||||
|
if queue == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var peers []link.PeerSeen
|
||||||
|
if ns != nil {
|
||||||
|
for _, p := range ns.Peers {
|
||||||
|
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var machines link.Machines
|
||||||
|
if c := machinesNow.Load(); c != nil {
|
||||||
|
machines.ByAddress, machines.Hub = c.Machines()
|
||||||
|
}
|
||||||
|
report := link.NodeHealthOf(queue.Membership.Node, h, peers, machines)
|
||||||
|
if queue.Fallback != nil && report.Fallback == nil {
|
||||||
|
report.Fallback = queue.Fallback() // node-account-fallback (ADR 0583 §6), as the written report carries it
|
||||||
|
}
|
||||||
|
_, err := queue.WriteNodeHealth(ctx, report)
|
||||||
|
// The findings the report says are raised by the engine here, at every look, written to the bus or not (design 47 G A-197).
|
||||||
|
if keeper := ownFindingsNow.Load(); keeper != nil {
|
||||||
|
keeper.Report(report, machines.ByAddress)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err != nil && err.Error() != *unwritten:
|
||||||
|
*unwritten = err.Error()
|
||||||
|
say("this machine's health report is not written to the bus: " + err.Error())
|
||||||
|
case err == nil && *unwritten != "":
|
||||||
|
*unwritten = ""
|
||||||
|
say("this machine's health report is written to the bus again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// healthAsReported is a statement as the report and the event carry it.
|
// healthAsReported is a statement as the report and the event carry it.
|
||||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||||
@@ -1466,7 +1462,7 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
|
|||||||
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
|
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
|
||||||
for _, p := range ns.Parts {
|
for _, p := range ns.Parts {
|
||||||
h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||||
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: p.Streak})
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: int64(p.Streak)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return h
|
return h
|
||||||
@@ -1640,6 +1636,15 @@ func announceOr(say link.Announce) func(string) {
|
|||||||
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
||||||
var applying sync.Mutex
|
var applying sync.Mutex
|
||||||
|
|
||||||
|
// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep,
|
||||||
|
// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files.
|
||||||
|
func objectKeep(opts options) *apply.ObjectKeep {
|
||||||
|
if opts.state == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects"))
|
||||||
|
}
|
||||||
|
|
||||||
// appliedModule is how the apply of one module's declaration came out.
|
// appliedModule is how the apply of one module's declaration came out.
|
||||||
type appliedModule struct {
|
type appliedModule struct {
|
||||||
// Refused is why nothing was applied, when nothing was.
|
// Refused is why nothing was applied, when nothing was.
|
||||||
@@ -1679,7 +1684,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
|
|||||||
|
|
||||||
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
||||||
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
||||||
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
|
if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||||
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1706,11 +1711,11 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
|
|||||||
func(known store.State) (apply.Report, store.State, error) {
|
func(known store.State) (apply.Report, store.State, error) {
|
||||||
if scope == nil {
|
if scope == nil {
|
||||||
return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
|
return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
|
||||||
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
opts.run(), announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
||||||
apply.WindowsIn(filepath.Dir(opts.state)))
|
apply.WindowsIn(filepath.Dir(opts.state)))
|
||||||
}
|
}
|
||||||
return apply.ApplyModule(ctx, built, declared, *scope, known, store.OriginDeclared,
|
return apply.ApplyModule(ctx, built, declared, *scope, known, store.OriginDeclared,
|
||||||
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
opts.run(), announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
||||||
apply.WindowsIn(filepath.Dir(opts.state)))
|
apply.WindowsIn(filepath.Dir(opts.state)))
|
||||||
}, announceOr(say))
|
}, announceOr(say))
|
||||||
|
|
||||||
@@ -1773,9 +1778,8 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
if j := judging; j != nil {
|
if j := judging; j != nil {
|
||||||
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
||||||
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
||||||
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
judgeWhatIsDeclared(declared, held)
|
||||||
st, _ := j.Look(ctx)
|
st, _ := j.Look(ctx)
|
||||||
netJudge.declare(declaredResolvConf(declared))
|
|
||||||
var netSt *network.Statement
|
var netSt *network.Statement
|
||||||
if n := netJudge.get(); n != nil {
|
if n := netJudge.get(); n != nil {
|
||||||
ns := n.Last()
|
ns := n.Last()
|
||||||
@@ -1783,7 +1787,6 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
}
|
}
|
||||||
var unitSt *units.Statement
|
var unitSt *units.Statement
|
||||||
if u := unitJudge; u != nil {
|
if u := unitJudge; u != nil {
|
||||||
u.Set(units.OwnedBy(declared, held))
|
|
||||||
us := u.Last()
|
us := u.Last()
|
||||||
unitSt = &us
|
unitSt = &us
|
||||||
}
|
}
|
||||||
@@ -1791,11 +1794,9 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
// said with it, relogin needed included, in the report that says the apply.
|
// said with it, relogin needed included, in the report that says the apply.
|
||||||
var accountSt *accounts.Statement
|
var accountSt *accounts.Statement
|
||||||
if a := accountJudge; a != nil {
|
if a := accountJudge; a != nil {
|
||||||
a.Set(accounts.Of(declared, held))
|
|
||||||
as, _ := a.Look(ctx)
|
as, _ := a.Look(ctx)
|
||||||
accountSt = &as
|
accountSt = &as
|
||||||
}
|
}
|
||||||
accesses.set(declared, held)
|
|
||||||
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
||||||
keeping.poke(ctx)
|
keeping.poke(ctx)
|
||||||
accessSt, _ := accesses.look(time.Now())
|
accessSt, _ := accesses.look(time.Now())
|
||||||
@@ -1890,6 +1891,46 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// judgeWhatIsDeclared hands every judge of the health loop what is declared here now: the long-running resources, the
|
||||||
|
// resolver file, the units, the accounts and the libraries. A resource, unit, account or library no longer declared is
|
||||||
|
// forgotten by its judge, so a module whose composition drops one is no longer watched for it.
|
||||||
|
func judgeWhatIsDeclared(declared *declaration.Declaration, held map[string]bool) {
|
||||||
|
if j := judging; j != nil {
|
||||||
|
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
||||||
|
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
||||||
|
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
||||||
|
}
|
||||||
|
netJudge.declare(declaredResolvConf(declared))
|
||||||
|
if u := unitJudge; u != nil {
|
||||||
|
u.Set(units.OwnedBy(declared, held))
|
||||||
|
}
|
||||||
|
if a := accountJudge; a != nil {
|
||||||
|
a.Set(accounts.Of(declared, held))
|
||||||
|
}
|
||||||
|
accesses.set(declared, held)
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeWhatRunsNow hands the judges what runs now, as the node agent holds it once a module settled (applied or
|
||||||
|
// removed) or was taken as running: the facts said inside an apply read the module's composition from before it, so a
|
||||||
|
// resource its new composition dropped stayed watched until the next facts (g14 and shanks, 2026-10-30 16:14: restic's
|
||||||
|
// process restic.measure, removed, then said down every look).
|
||||||
|
func judgeWhatRunsNow(statePath string, signer ed25519.PublicKey) {
|
||||||
|
if judging == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
declared, err := declaredNow(statePath, signer)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
held := map[string]bool{}
|
||||||
|
if known, err := store.Load(statePath); err == nil {
|
||||||
|
for _, h := range known.Held {
|
||||||
|
held[h.ID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
judgeWhatIsDeclared(declared, held)
|
||||||
|
}
|
||||||
|
|
||||||
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs
|
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs
|
||||||
// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it
|
// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it
|
||||||
// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's.
|
// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's.
|
||||||
@@ -2041,7 +2082,7 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
|
say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
|
||||||
os.Exit(0)
|
exitBetweenApplies(say)
|
||||||
}
|
}
|
||||||
|
|
||||||
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
|
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
|
||||||
@@ -2131,3 +2172,57 @@ func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[st
|
|||||||
}
|
}
|
||||||
return missingModules(fresh), now
|
return missingModules(fresh), now
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newlyInactive are the modules of the resources a statement says down (not running after their grace, but still known to
|
||||||
|
// their manager, and not restarting) that the last statement acted on did not, each once, sorted, and the resources down
|
||||||
|
// now: a resource still down is not acted on again, and one that came back and went down again is (ADR 0568 §2).
|
||||||
|
func newlyInactive(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
|
||||||
|
now := map[string]bool{}
|
||||||
|
var fresh liveness.Statement
|
||||||
|
for _, r := range st.Resources {
|
||||||
|
if r.State != liveness.Unhealthy || r.Reason != liveness.ReasonDown || r.Missing {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
now[r.ID] = true
|
||||||
|
if !last[r.ID] {
|
||||||
|
fresh.Resources = append(fresh.Resources, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return missingModules(withMissing(fresh)), now
|
||||||
|
}
|
||||||
|
|
||||||
|
// withMissing is the statement with each of its resources marked missing, so missingModules lists their modules.
|
||||||
|
func withMissing(st liveness.Statement) liveness.Statement {
|
||||||
|
for i := range st.Resources {
|
||||||
|
st.Resources[i].Missing = true
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
||||||
|
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
||||||
|
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
||||||
|
func recordKeys(state string, trusts func() []string) func() link.RecordKeys {
|
||||||
|
return func() link.RecordKeys {
|
||||||
|
var k link.RecordKeys
|
||||||
|
if trusts != nil {
|
||||||
|
k.Trusts = trusts()
|
||||||
|
}
|
||||||
|
if id, err := identity.Load(identity.Path(state)); err == nil {
|
||||||
|
k.TunnelKey = id.Overlay.Public
|
||||||
|
}
|
||||||
|
if sk, err := identity.LoadSealingKey(identity.SealingKeyPath(state)); err == nil {
|
||||||
|
k.SealingKey = sk.Public
|
||||||
|
}
|
||||||
|
if sv, err := identity.LoadServingKey(identity.ServingKeyPath(state)); err == nil {
|
||||||
|
k.ServingKey = sv.Public
|
||||||
|
}
|
||||||
|
if cert, err := os.ReadFile(identity.CertificatePath(state)); err == nil {
|
||||||
|
k.Certificate = string(cert)
|
||||||
|
}
|
||||||
|
if authority, err := os.ReadFile(authorityPath(state)); err == nil {
|
||||||
|
k.Authority = string(authority)
|
||||||
|
}
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -132,6 +133,57 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
|
|||||||
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
||||||
// what the other recorded.
|
// what the other recorded.
|
||||||
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
||||||
|
// Issue 1585: the engine's stop waits for the apply in flight to end and save, and then lets no save start; before,
|
||||||
|
// the process ended under it and a save cut before its rename left its temporary file and the state before it.
|
||||||
|
t.Run("the stop waits for the apply in flight and its save", func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
applying.Lock()
|
||||||
|
var said []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
say := func(line string) {
|
||||||
|
mu.Lock()
|
||||||
|
said = append(said, line)
|
||||||
|
mu.Unlock()
|
||||||
|
if strings.Contains(line, "waiting up to") {
|
||||||
|
close(waiting)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stopped := make(chan func(), 1)
|
||||||
|
go func() { stopped <- stopBetweenApplies(say, time.Hour) }()
|
||||||
|
// Said before it blocks, and it cannot end before the apply lets go: so it has not ended now.
|
||||||
|
<-waiting
|
||||||
|
select {
|
||||||
|
case release := <-stopped:
|
||||||
|
release()
|
||||||
|
applying.Unlock()
|
||||||
|
t.Fatal("the stop did not wait for the apply in flight")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
applying.Unlock()
|
||||||
|
release := <-stopped
|
||||||
|
saved := make(chan error, 1)
|
||||||
|
go func() { saved <- store.Save(path, store.State{}) }()
|
||||||
|
// A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look.
|
||||||
|
select {
|
||||||
|
case <-saved:
|
||||||
|
release()
|
||||||
|
t.Fatal("a save started after the stop")
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
if err := <-saved; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") {
|
||||||
|
t.Errorf("said %q", said)
|
||||||
|
}
|
||||||
|
})
|
||||||
// A host is built for one system at link time, and a test binary has no link time: this asks
|
// A host is built for one system at link time, and a test binary has no link time: this asks
|
||||||
// the machine it runs on, and stands aside where the answer is no.
|
// the machine it runs on, and stands aside where the answer is no.
|
||||||
built, err := system.For("arch")
|
built, err := system.For("arch")
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// markWithin bounds one marking of the haves (what the composer tells its agent: a refusal, the words it keeps). The marks
|
||||||
|
// read and write each module's have over the bus one after the other; a bus that answers slowly or not at all must hold
|
||||||
|
// the marking, never the composition pass (issue 1965, shanks 2026-11-05 22:22: a pass waited minutes on them).
|
||||||
|
var markWithin = 45 * time.Second
|
||||||
|
|
||||||
|
// marker runs the composer's marks of the haves off the composition pass: set returns at once, one worker runs what was
|
||||||
|
// set last under each name, in the order the names were first set, each within markWithin. A mark set while one runs
|
||||||
|
// replaces the pending mark of its name, so the haves are marked once for the newest word, not once for every pass.
|
||||||
|
type marker struct {
|
||||||
|
run context.Context
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]func(context.Context)
|
||||||
|
order []string
|
||||||
|
working bool
|
||||||
|
idle *sync.Cond
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMarker(run context.Context) *marker {
|
||||||
|
m := &marker{run: run, pending: map[string]func(context.Context){}}
|
||||||
|
m.idle = sync.NewCond(&m.mu)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// set takes the mark name should carry; it never waits on the bus.
|
||||||
|
func (m *marker) set(name string, mark func(context.Context)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if _, queued := m.pending[name]; !queued {
|
||||||
|
m.order = append(m.order, name)
|
||||||
|
}
|
||||||
|
m.pending[name] = mark
|
||||||
|
if m.working {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.working = true
|
||||||
|
go m.work()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *marker) work() {
|
||||||
|
for {
|
||||||
|
m.mu.Lock()
|
||||||
|
if len(m.order) == 0 || m.run.Err() != nil {
|
||||||
|
m.order, m.pending = nil, map[string]func(context.Context){}
|
||||||
|
m.working = false
|
||||||
|
m.idle.Broadcast()
|
||||||
|
m.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name := m.order[0]
|
||||||
|
m.order = m.order[1:]
|
||||||
|
mark := m.pending[name]
|
||||||
|
delete(m.pending, name)
|
||||||
|
m.mu.Unlock()
|
||||||
|
ctx, cancel := context.WithTimeout(m.run, markWithin)
|
||||||
|
mark(ctx)
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait returns once no mark is pending or running (for tests).
|
||||||
|
func (m *marker) wait() {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
for m.working {
|
||||||
|
m.idle.Wait()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1965 (shanks 2026-11-05 22:22): the marks of the haves never hold the composition pass. A mark that stalls on the
|
||||||
|
// bus is cut at markWithin, a mark set while one runs replaces the pending one of its name, and set itself never waits.
|
||||||
|
func marksOfTheHavesNeverHoldThePass(t *testing.T) {
|
||||||
|
// The bound is the test's own: the stalled mark is released by a channel, never by a real wait.
|
||||||
|
setMarkWithin(t, time.Hour)
|
||||||
|
m := newMarker(context.Background())
|
||||||
|
release := make(chan struct{})
|
||||||
|
var ran []string
|
||||||
|
stalled := make(chan struct{})
|
||||||
|
m.set("refused", func(ctx context.Context) { // a have write that stalls until the test lets it go
|
||||||
|
close(stalled)
|
||||||
|
<-release
|
||||||
|
ran = append(ran, "refused-stalled")
|
||||||
|
})
|
||||||
|
<-stalled
|
||||||
|
// Set while the first runs: only the newest of a name is kept.
|
||||||
|
m.set("kept", func(context.Context) { ran = append(ran, "kept-old") })
|
||||||
|
m.set("kept", func(context.Context) { ran = append(ran, "kept-new") })
|
||||||
|
m.set("refused", func(context.Context) { ran = append(ran, "refused-new") })
|
||||||
|
// All four sets above returned while the first mark still stalled: set never waits on it.
|
||||||
|
close(release)
|
||||||
|
m.wait()
|
||||||
|
want := []string{"refused-stalled", "kept-new", "refused-new"}
|
||||||
|
if len(ran) != len(want) {
|
||||||
|
t.Fatalf("ran %v; want %v", ran, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if ran[i] != want[i] {
|
||||||
|
t.Fatalf("ran %v; want %v", ran, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMarkWithin(t *testing.T, d time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
old := markWithin
|
||||||
|
markWithin = d
|
||||||
|
t.Cleanup(func() { markWithin = old })
|
||||||
|
}
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/meshcli"
|
|
||||||
)
|
|
||||||
|
|
||||||
// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account
|
|
||||||
// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served
|
|
||||||
// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer.
|
|
||||||
func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) {
|
|
||||||
if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil {
|
|
||||||
say("mesh-cli is not answered on this machine: " + err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// forwardOver asks the controller over the queue's link, and follows the line to what came of it.
|
|
||||||
func forwardOver(queue *link.Session) meshcli.Forward {
|
|
||||||
return func(ctx context.Context, asked meshcli.Asked) cli.Answer {
|
|
||||||
return meshcli.AskTheController(ctx, queue.AskController, asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/crypto/nacl/box"
|
"golang.org/x/crypto/nacl/box"
|
||||||
@@ -40,6 +41,11 @@ func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
||||||
}
|
}
|
||||||
|
if moved.Superseded != "" {
|
||||||
|
if _, said := supersededSaid.LoadOrStore(moved.Secret, true); !said {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: %s\n", moved.Superseded)
|
||||||
|
}
|
||||||
|
}
|
||||||
return moved, err
|
return moved, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -104,8 +110,30 @@ func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret)
|
|||||||
Generation int `json:"generation"`
|
Generation int `json:"generation"`
|
||||||
Unchanged bool `json:"unchanged"`
|
Unchanged bool `json:"unchanged"`
|
||||||
ReadBack string `json:"read_back"`
|
ReadBack string `json:"read_back"`
|
||||||
|
// Refused is the vault's refusal as fields (mesh-vault's MoveRefused, mesh-catalog f3b13de2): nothing was taken.
|
||||||
|
Refused *struct {
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
Held *heldValue `json:"held"`
|
||||||
|
} `json:"refused"`
|
||||||
|
Why string `json:"why"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" {
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's answer for %s does not read: %v", opened.Secret, err)
|
||||||
|
}
|
||||||
|
if r := kept.Refused; r != nil {
|
||||||
|
// The vault holds another value for the name, and the move never replaces one: the vault's value is the live
|
||||||
|
// one, and this machine's older value is superseded, never asked again (the sweep of 2026-10-30: it failed
|
||||||
|
// "answered with no read-back" on every ask).
|
||||||
|
if r.Reason == vaultHoldsAnother && r.Held != nil && r.Held.Name != "" {
|
||||||
|
return link.MovedSecret{Secret: r.Held.Name, Generation: r.Held.Generation, Superseded: r.Held.words()}, nil
|
||||||
|
}
|
||||||
|
why := kept.Why
|
||||||
|
if why == "" {
|
||||||
|
why = r.Reason
|
||||||
|
}
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault refused %s: %s", opened.Secret, why)
|
||||||
|
}
|
||||||
|
if kept.Name == "" || kept.ReadBack == "" {
|
||||||
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
||||||
}
|
}
|
||||||
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
||||||
@@ -124,6 +152,31 @@ func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret)
|
|||||||
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// vaultHoldsAnother is the vault's reason when it holds another value for the name (mesh-vault RefusedHeldAnother).
|
||||||
|
const vaultHoldsAnother = "holds-another-value"
|
||||||
|
|
||||||
|
// heldValue is the record of the value the vault holds instead, never the value (mesh-vault HeldAnother).
|
||||||
|
type heldValue struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
ChangedBy string `json:"changed_by"`
|
||||||
|
ChangedAt time.Time `json:"changed_at"`
|
||||||
|
Origin string `json:"origin"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// words say the held value's record, for the journal and the controller.
|
||||||
|
func (h heldValue) words() string {
|
||||||
|
w := fmt.Sprintf("the vault holds another value for %s, generation %d, changed by %s at %s", h.Name, h.Generation,
|
||||||
|
h.ChangedBy, h.ChangedAt.UTC().Format(time.RFC3339))
|
||||||
|
if h.Origin != "" {
|
||||||
|
w += ", origin " + h.Origin
|
||||||
|
}
|
||||||
|
return w + ": that value is the live one, and this machine's older value is superseded and not moved"
|
||||||
|
}
|
||||||
|
|
||||||
|
// supersededSaid holds the names a superseded move was said of, so the journal says each once.
|
||||||
|
var supersededSaid sync.Map
|
||||||
|
|
||||||
func wipe(b []byte) {
|
func wipe(b []byte) {
|
||||||
for i := range b {
|
for i := range b {
|
||||||
b[i] = 0
|
b[i] = 0
|
||||||
|
|||||||
@@ -17,13 +17,15 @@ import (
|
|||||||
// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
|
// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
|
||||||
// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
|
// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
|
||||||
type standInVault struct {
|
type standInVault struct {
|
||||||
t *testing.T
|
t *testing.T
|
||||||
pub *[32]byte
|
pub *[32]byte
|
||||||
priv *[32]byte
|
priv *[32]byte
|
||||||
opened map[string]any
|
opened map[string]any
|
||||||
kept string
|
kept string
|
||||||
lie string
|
lie string
|
||||||
refuse string
|
refuse string
|
||||||
|
// answer replaces the second step's answer: the vault's refusal as fields (mesh-catalog f3b13de2).
|
||||||
|
answer map[string]any
|
||||||
carried []string
|
carried []string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,6 +53,9 @@ func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
|
|||||||
v.t.Fatal("what the engine sealed does not open with the call's key")
|
v.t.Fatal("what the engine sealed does not open with the call's key")
|
||||||
}
|
}
|
||||||
v.kept = string(value)
|
v.kept = string(value)
|
||||||
|
if v.answer != nil {
|
||||||
|
return json.Marshal(v.answer)
|
||||||
|
}
|
||||||
back := v.kept
|
back := v.kept
|
||||||
if v.lie != "" {
|
if v.lie != "" {
|
||||||
back = v.lie
|
back = v.lie
|
||||||
@@ -119,6 +124,28 @@ func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
|
|||||||
!strings.Contains(err.Error(), "never replaces") {
|
!strings.Contains(err.Error(), "never replaces") {
|
||||||
t.Fatalf("the vault's refusal: %v", err)
|
t.Fatalf("the vault's refusal: %v", err)
|
||||||
}
|
}
|
||||||
|
// The vault's refusal as fields (mesh-catalog f3b13de2, no read-back): for another value held, the vault's is the
|
||||||
|
// live one and the move is done, superseded, at the held generation; any other reason fails in the vault's words.
|
||||||
|
held := map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 3,
|
||||||
|
"changed_by": "mesh-vault@novox", "changed_at": "2026-10-30T15:50:00Z", "origin": "made"}
|
||||||
|
for _, row := range []struct {
|
||||||
|
refused map[string]any
|
||||||
|
gen int
|
||||||
|
fails string
|
||||||
|
}{
|
||||||
|
{map[string]any{"reason": "holds-another-value", "held": held}, 3, ""},
|
||||||
|
{map[string]any{"reason": "locked"}, 0, "the vault refused"},
|
||||||
|
} {
|
||||||
|
v := &standInVault{t: t, answer: map[string]any{"refused": row.refused, "why": "in words"}}
|
||||||
|
moved, err := moveIntoTheVault(v.ask, key, aMoveOf(t, key))
|
||||||
|
switch {
|
||||||
|
case row.fails != "" && (err == nil || !strings.Contains(err.Error(), row.fails)):
|
||||||
|
t.Errorf("refused %v: %v", row.refused["reason"], err)
|
||||||
|
case row.fails == "" && (err != nil || moved.Generation != row.gen || moved.Secret != held["name"] ||
|
||||||
|
!strings.Contains(moved.Superseded, "changed by mesh-vault@novox")):
|
||||||
|
t.Errorf("refused %v: moved %+v, %v", row.refused["reason"], moved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
// A pair credential names both its ends.
|
// A pair credential names both its ends.
|
||||||
pair := aMoveOf(t, key)
|
pair := aMoveOf(t, key)
|
||||||
pair.Module, pair.Name = "", ""
|
pair.Module, pair.Name = "", ""
|
||||||
|
|||||||
+581
-82
@@ -3,22 +3,32 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
"github.com/novox/mesh-host/internal/agent"
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/compose/frombus"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/seatwait"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
|
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
|
||||||
@@ -35,6 +45,20 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return w.updates(ctx, in), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WatchAwait is Watch for a state that its holder makes when it starts (the assignment seat's): a bucket not on the bus
|
||||||
|
// yet is waited for on the bus's word that it was created (link.Session.KVWatchAwait), and waiting is told why.
|
||||||
|
func (w bucketWatch) WatchAwait(ctx context.Context, pattern string, waiting func(error)) (<-chan agent.Update, error) {
|
||||||
|
in, err := w.session.KVWatchAwait(ctx, w.bucket, pattern, waiting)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return w.updates(ctx, in), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w bucketWatch) updates(ctx context.Context, in <-chan link.KVUpdate) <-chan agent.Update {
|
||||||
out := make(chan agent.Update, 64)
|
out := make(chan agent.Update, 64)
|
||||||
go func() {
|
go func() {
|
||||||
defer close(out)
|
defer close(out)
|
||||||
@@ -46,34 +70,62 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
return out, nil
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// sessionHaves writes the haves the agent says, through the session.
|
// sessionHaves writes the haves the agent says, through the session.
|
||||||
type sessionHaves struct{ session *link.Session }
|
type sessionHaves struct{ session *link.Session }
|
||||||
|
|
||||||
func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
|
func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
|
||||||
|
if module == agent.NodeModule {
|
||||||
|
// The node's own key: the digest of its own composition it runs, and what that carries sealed.
|
||||||
|
got, ok := h.session.ReadNodeHave(ctx)
|
||||||
|
if !ok || got.Composed == "" {
|
||||||
|
return agent.Have{}, false
|
||||||
|
}
|
||||||
|
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true
|
||||||
|
}
|
||||||
got, ok := h.session.ReadHave(ctx, module)
|
got, ok := h.session.ReadHave(ctx, module)
|
||||||
if !ok {
|
if !ok {
|
||||||
return agent.Have{}, false
|
return agent.Have{}, false
|
||||||
}
|
}
|
||||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail}, true
|
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||||
|
Sealed: got.SealedAt, Wait: got.Wait}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
|
||||||
|
func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, bool, error) {
|
||||||
|
if module == agent.NodeModule {
|
||||||
|
// The node's own key, its read the bus did not answer told as such (issue 1126, 2026-10-28).
|
||||||
|
got, ok, err := h.session.ReadNodeHaveErr(ctx)
|
||||||
|
if !ok || err != nil || got.Composed == "" {
|
||||||
|
return agent.Have{}, false, err
|
||||||
|
}
|
||||||
|
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true, nil
|
||||||
|
}
|
||||||
|
got, ok, err := h.session.ReadHaveErr(ctx, module)
|
||||||
|
if !ok || err != nil {
|
||||||
|
return agent.Have{}, false, err
|
||||||
|
}
|
||||||
|
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||||
|
Sealed: got.SealedAt, Wait: got.Wait}, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||||
if x.Module == agent.NodeModule {
|
if x.Module == agent.NodeModule {
|
||||||
// The node's own key holds its facts and the digest of the composition it runs.
|
// The node's own key holds its facts and the digest of the composition it runs.
|
||||||
if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) {
|
if x.Phase == "" || x.Phase == string(nodeengineclient.PhaseRunning) {
|
||||||
h.session.NoteNode(ctx, x.Composed)
|
h.session.NoteNode(ctx, x.Composed, x.Sealed)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
phase := x.Phase
|
phase := x.Phase
|
||||||
if phase == string(delivery.PhaseRunning) {
|
if phase == string(nodeengineclient.PhaseRunning) {
|
||||||
phase = ""
|
phase = ""
|
||||||
}
|
}
|
||||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail)})
|
Failed: phase == string(nodeengineclient.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||||
|
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait, Wait: x.Wait})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||||
@@ -117,19 +169,54 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
|
|||||||
defer stop()
|
defer stop()
|
||||||
// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
|
// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
|
||||||
// 1945). An answer that cannot be had or read asks for the backup as before.
|
// 1945). An answer that cannot be had or read asks for the backup as before.
|
||||||
if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
|
ask := func(verb string) func(context.Context) (json.RawMessage, error) {
|
||||||
|
return func(ctx context.Context) (json.RawMessage, error) {
|
||||||
|
return session.AskSeat(ctx, "node-backup", verb, map[string]any{"module": module}, 30*time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if said, err := askUntilHeard(ctx, ask("backed-up"), backupAskFor, link.Backoff); err == nil {
|
||||||
if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
|
if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
|
||||||
return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
|
return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
asked := time.Now()
|
asked := time.Now()
|
||||||
if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
|
if _, err := askUntilHeard(ctx, ask("now"), backupAskFor, link.Backoff); err != nil {
|
||||||
return agent.Backup{Words: err.Error()}
|
return agent.Backup{Words: err.Error()}
|
||||||
}
|
}
|
||||||
return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
|
return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backupAskFor is how long the backup's ask is asked again while it is known never to have reached the holder of
|
||||||
|
// node-backup: nothing on this machine answers it (the tool runner restarting, its subscriptions not made yet) or no
|
||||||
|
// link is open. The sweep of 2026-10-30 met it on ace's redis, postgres and restic (14:03), step-ca (13:55) and
|
||||||
|
// mesh-vault on novox (15:22): each was held on one unanswered ask. Past the bound the backup failed, said with how
|
||||||
|
// often and how long it was asked: the module is held, its have says so, and the assigner asks the operator.
|
||||||
|
var backupAskFor = 3 * time.Minute
|
||||||
|
|
||||||
|
// askUntilHeard asks, and asks again after wait(try) while the answer says the ask was never delivered (link.ErrNotAsked),
|
||||||
|
// for at most within: an ask that may have reached its holder (a timeout, a refusal in words) is never asked twice, so a
|
||||||
|
// backup is never started twice.
|
||||||
|
func askUntilHeard(ctx context.Context, ask func(context.Context) (json.RawMessage, error), within time.Duration,
|
||||||
|
wait func(int) time.Duration) (json.RawMessage, error) {
|
||||||
|
began := time.Now()
|
||||||
|
for try := 1; ; try++ {
|
||||||
|
said, err := ask(ctx)
|
||||||
|
if err == nil || !errors.Is(err, link.ErrNotAsked) {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
pause := wait(try)
|
||||||
|
if time.Since(began)+pause > within {
|
||||||
|
return nil, fmt.Errorf("asked %d times over %s and never heard: %w", try, time.Since(began).Round(time.Second), err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("the node agent was stopped while it asked again: %w", err)
|
||||||
|
case <-time.After(pause):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
|
// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
|
||||||
// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
|
// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
|
||||||
// or module, or from before the ask, is not the answer.
|
// or module, or from before the ask, is not the answer.
|
||||||
@@ -139,7 +226,7 @@ func awaitBackup(ctx context.Context, events <-chan []byte, node, module string,
|
|||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return agent.Backup{Words: "the node agent was stopped while the backup ran"}
|
return agent.Backup{Words: agent.BackupInterrupted, Interrupted: true}
|
||||||
case <-bound.C:
|
case <-bound.C:
|
||||||
return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)}
|
return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)}
|
||||||
case raw := <-events:
|
case raw := <-events:
|
||||||
@@ -173,16 +260,16 @@ func awaitBackup(ctx context.Context, events <-chan []byte, node, module string,
|
|||||||
// the controller put modules it knew no fingerprint of the new form for into the node's own composition while their
|
// the controller put modules it knew no fingerprint of the new form for into the node's own composition while their
|
||||||
// wants named their own, and the declaration joined of both declared each such file twice, which the filling refuses
|
// wants named their own, and the declaration joined of both declared each such file twice, which the filling refuses
|
||||||
// ("2 resources declare X") on every node. The node's own copy is left out, and which were left out is said once.
|
// ("2 resources declare X") on every node. The node's own copy is left out, and which were left out is said once.
|
||||||
func declarationOf(cs ...delivery.Composed) (*declaration.Declaration, error) {
|
func declarationOf(cs ...compositions.Composed) (*declaration.Declaration, error) {
|
||||||
d := &declaration.Declaration{Version: declaration.Version}
|
d := &declaration.Declaration{Version: declaration.Version}
|
||||||
var modules []delivery.Composed
|
var modules []compositions.Composed
|
||||||
for _, c := range cs {
|
for _, c := range cs {
|
||||||
if c.Module != "" {
|
if c.Module != "" {
|
||||||
modules = append(modules, c)
|
modules = append(modules, c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(modules) > 0 {
|
if len(modules) > 0 {
|
||||||
joined := make([]delivery.Composed, 0, len(cs))
|
joined := make([]compositions.Composed, 0, len(cs))
|
||||||
for _, c := range cs {
|
for _, c := range cs {
|
||||||
if c.Module == "" {
|
if c.Module == "" {
|
||||||
var dropped []string
|
var dropped []string
|
||||||
@@ -217,7 +304,7 @@ func declarationOf(cs ...delivery.Composed) (*declaration.Declaration, error) {
|
|||||||
// the same kind and id. A resource that shares only its target (a unit or container of the same name, another id) is
|
// the same kind and id. A resource that shares only its target (a unit or container of the same name, another id) is
|
||||||
// not the module's, and stays: dropped, its record would be an orphan the apply removes, stopping the module's unit. It
|
// not the module's, and stays: dropped, its record would be an orphan the apply removes, stopping the module's unit. It
|
||||||
// answers what it left out, as "<id> (<module>)".
|
// answers what it left out, as "<id> (<module>)".
|
||||||
func ownWithout(own delivery.Composed, modules []delivery.Composed) (delivery.Composed, []string) {
|
func ownWithout(own compositions.Composed, modules []compositions.Composed) (compositions.Composed, []string) {
|
||||||
byID := map[string]string{}
|
byID := map[string]string{}
|
||||||
for _, c := range modules {
|
for _, c := range modules {
|
||||||
if c.Module == "" {
|
if c.Module == "" {
|
||||||
@@ -273,14 +360,14 @@ func sayDroppedOnce(dropped []string) {
|
|||||||
"declares, and they are left out of it: %s (issue 1945)\n", len(dropped), key)
|
"declares, and they are left out of it: %s (issue 1945)\n", len(dropped), key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func scopeName(c delivery.Composed) string {
|
func scopeName(c compositions.Composed) string {
|
||||||
if c.Module == "" {
|
if c.Module == "" {
|
||||||
return agent.NodeModule
|
return agent.NodeModule
|
||||||
}
|
}
|
||||||
return c.Module
|
return c.Module
|
||||||
}
|
}
|
||||||
|
|
||||||
func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
func resourcesOf(cs []compositions.Composed) []declaration.Resource {
|
||||||
var out []declaration.Resource
|
var out []declaration.Resource
|
||||||
for _, c := range cs {
|
for _, c := range cs {
|
||||||
for _, raw := range c.Resources {
|
for _, raw := range c.Resources {
|
||||||
@@ -292,48 +379,17 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a
|
|
||||||
// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the
|
|
||||||
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
|
||||||
const operatorKeysFile = "operator-keys"
|
|
||||||
|
|
||||||
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
|
|
||||||
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
|
|
||||||
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
|
|
||||||
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
|
|
||||||
return func() []ed25519.PublicKey {
|
|
||||||
keys := []ed25519.PublicKey{}
|
|
||||||
if len(controller) == ed25519.PublicKeySize {
|
|
||||||
keys = append(keys, controller)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
|
||||||
if err != nil {
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
if line == "" || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key, err := base64.StdEncoding.DecodeString(line)
|
|
||||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
keys = append(keys, ed25519.PublicKey(key))
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
||||||
// none, ADR 0539 §9) and its filled paths.
|
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
|
||||||
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
var by map[string][]json.RawMessage
|
var by map[string][]json.RawMessage
|
||||||
var filled map[string][]string
|
var filled map[string][]string
|
||||||
return func() (map[string][]json.RawMessage, map[string][]string) {
|
return func() (map[string][]json.RawMessage, map[string][]string) {
|
||||||
once.Do(func() {
|
once.Do(func() {
|
||||||
|
if keptRetired.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@@ -383,7 +439,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
|
|||||||
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
||||||
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
||||||
// account may not read) is said, and tried again with a growing wait, as the link is.
|
// account may not read) is said, and tried again with a growing wait, as the link is.
|
||||||
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
|
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler,
|
||||||
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
||||||
wait := 2 * time.Second
|
wait := 2 * time.Second
|
||||||
for ctx.Err() == nil {
|
for ctx.Err() == nil {
|
||||||
@@ -393,17 +449,46 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}
|
}
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
node := session.Membership.Node
|
node := session.Membership.Node
|
||||||
composedBucket, err := delivery.ComposedBucketFor(node)
|
var keptFetch atomic.Pointer[apply.KeptFetch] // the composer's, set once it exists
|
||||||
|
composedBucket, err := compositions.BucketFor(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cancel()
|
cancel()
|
||||||
say("the node agent cannot name this node's composition bucket: " + err.Error())
|
say("the node agent cannot name this node's composition bucket: " + err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var ag *agent.Agent
|
var ag *agent.Agent
|
||||||
|
// The kept release-5 declaration is no longer read once every module the seat's assignment puts here has a have of its
|
||||||
|
// own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write).
|
||||||
|
// Nor before this engine keeps the node's own as it runs (applied/node.json): until then the kept declaration is the one
|
||||||
|
// record of what the node's own composition carries, its folded modules among them (issue 774, review of the
|
||||||
|
// re-land, round 1: novox's first start would compose its own with nats running nothing).
|
||||||
|
if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Load() {
|
||||||
|
if !appliedStoreFor(opts.state).holds(agent.NodeModule) {
|
||||||
|
say("the kept declaration is still read: " + why + ", but this engine keeps no node's own composition yet")
|
||||||
|
} else if !keptRetired.Swap(true) {
|
||||||
|
say("the kept declaration is no longer read: " + why)
|
||||||
|
}
|
||||||
|
}
|
||||||
kept := keptOnce(opts, signer)
|
kept := keptOnce(opts, signer)
|
||||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||||
// node, and a node whose compositions carry no list writes nothing.
|
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
if byModule, _ := kept(); byModule != nil {
|
||||||
|
for module, resources := range byModule {
|
||||||
|
if err := keepAuthority(opts.state, compositions.Composed{Module: module, Resources: resources}); err != nil {
|
||||||
|
say("the mesh's certificate the kept declaration carries could not be kept for this machine's record: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the
|
||||||
|
// bus module this node runs declares, as its agent holds what runs (its own composition, or the declaration kept
|
||||||
|
// before the switch): never read from the node's composition bucket, which the controller wrote. On every node;
|
||||||
|
// a node that runs no list writes nothing. Looked at again whenever what runs changed (RunsChanged, Settled).
|
||||||
|
users := userListFollower(opts, session, func() []compositions.Composed {
|
||||||
|
if now := current.get(); now != nil {
|
||||||
|
return now.Everything()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}, say, func() {
|
||||||
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
||||||
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
||||||
time.AfterFunc(reloadSettles, func() {
|
time.AfterFunc(reloadSettles, func() {
|
||||||
@@ -412,8 +497,6 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
// A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9).
|
|
||||||
vaultKey.following(users.Again)
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := users.Run(linked); err != nil && linked.Err() == nil {
|
if err := users.Run(linked); err != nil && linked.Err() == nil {
|
||||||
say("the bus's user list is not followed on this link: " + err.Error())
|
say("the bus's user list is not followed on this link: " + err.Error())
|
||||||
@@ -421,13 +504,23 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}()
|
}()
|
||||||
env := agent.Env{
|
env := agent.Env{
|
||||||
Node: node,
|
Node: node,
|
||||||
Wants: bucketWatch{session, delivery.AssignerWantsBucket},
|
Wants: bucketWatch{session, buckets.Wants},
|
||||||
Composed: bucketWatch{session, composedBucket},
|
Composed: bucketWatch{session, composedBucket},
|
||||||
Haves: sessionHaves{session},
|
Haves: sessionHaves{session},
|
||||||
Kept: func() map[string][]json.RawMessage { by, _ := kept(); return by },
|
Kept: func() map[string][]json.RawMessage { by, _ := kept(); return by },
|
||||||
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
||||||
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
||||||
Recorded: recordedIn(opts.state, say),
|
Recorded: recordedIn(opts.state, say),
|
||||||
|
// What this engine applied, kept on the machine and read at the start, so what runs is known before the node
|
||||||
|
// composes its own (issue 774).
|
||||||
|
Applied: appliedStoreFor(opts.state),
|
||||||
|
Inactive: inactiveIn(&liveness.Exec{Run: liveness.Runner(opts.run()), System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
||||||
|
Held: func() map[string]string {
|
||||||
|
if c := composerNow.Load(); c != nil {
|
||||||
|
return c.LastHeld()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -439,18 +532,45 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
},
|
},
|
||||||
Trusted: trustedKeys(opts, signer),
|
Backup: backupNode(session),
|
||||||
Backup: backupNode(session),
|
|
||||||
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
||||||
// again (ace, 2026-10-25 21:17).
|
// again (ace, 2026-10-25 21:17).
|
||||||
Settled: func(module string) { knockFilling(module, say) },
|
Settled: func(module string) {
|
||||||
Say: say,
|
knockFilling(module+" settled", say)
|
||||||
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
users.RunsChanged()
|
||||||
|
},
|
||||||
|
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
|
||||||
|
// a refusal from the engine's start does not wait for some module to settle (issue 924).
|
||||||
|
RunsChanged: func(why string) {
|
||||||
|
knockFilling(why, say)
|
||||||
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
users.RunsChanged()
|
||||||
|
},
|
||||||
|
Say: say,
|
||||||
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
||||||
Backoff: link.Backoff,
|
Backoff: link.Backoff,
|
||||||
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
|
Apply: func(ctx context.Context, c compositions.Composed, others []compositions.Composed) agent.Result {
|
||||||
actx, done := applyContext(session.WithLink(ctx))
|
actx, done := applyContext(session.WithLink(ctx))
|
||||||
res := applyComposition(actx, opts, c, others, sched, say)
|
if kf := keptFetch.Load(); kf != nil {
|
||||||
|
actx = apply.WithKept(actx, *kf)
|
||||||
|
}
|
||||||
|
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||||
|
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||||
|
held, herr := users.Hold(c)
|
||||||
|
var res agent.Result
|
||||||
|
if herr != nil {
|
||||||
|
res = failedWith(herr.Error())
|
||||||
|
} else {
|
||||||
|
res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say)
|
||||||
|
}
|
||||||
done()
|
done()
|
||||||
|
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
||||||
|
if !res.Failed {
|
||||||
|
if err := keepAuthority(opts.state, c); err != nil {
|
||||||
|
say("the mesh's certificate in " + c.Module + "'s composition could not be kept for this machine's record: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
// An apply of the bus module may write an older list than the newest composition holds: looked at again.
|
// An apply of the bus module may write an older list than the newest composition holds: looked at again.
|
||||||
if module, _ := users.State(); module != "" && module == c.Module {
|
if module, _ := users.State(); module != "" && module == c.Module {
|
||||||
users.Again()
|
users.Again()
|
||||||
@@ -459,20 +579,146 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
sayFacts(ctx, ag)
|
sayFacts(ctx, ag)
|
||||||
return res
|
return res
|
||||||
},
|
},
|
||||||
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
|
Stop: func(ctx context.Context, module string) agent.Result {
|
||||||
|
return stopComposition(ctx, opts, module)
|
||||||
|
},
|
||||||
|
Remove: func(ctx context.Context, module string, others []compositions.Composed) agent.Result {
|
||||||
actx, done := applyContext(session.WithLink(ctx))
|
actx, done := applyContext(session.WithLink(ctx))
|
||||||
res := removeComposition(actx, opts, module, others, sched, say)
|
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
|
||||||
done()
|
done()
|
||||||
afterApply()
|
afterApply()
|
||||||
sayFacts(ctx, ag)
|
sayFacts(ctx, ag)
|
||||||
return res
|
return res
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
// The node composes its own modules (novox/hq ADR 0547 §2, ADR 0549, issue 1126).
|
||||||
|
run, stopRun := context.WithCancel(linked)
|
||||||
|
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
|
||||||
|
// told when the node cannot compose, so each wanted module's have says it.
|
||||||
|
// The agent of this run alone: its composer tells it, never the agent of a run begun since (issue 1126: a pass in
|
||||||
|
// flight at the switch marked the next agent's modules failed).
|
||||||
|
own := &agentNow{}
|
||||||
|
// And the seat list it reads through the seat mesh-catalog, from its bootstrapped definitions until it has: the
|
||||||
|
// composer hands nothing new meanwhile, and the node says it has waited five minutes (novox/hq ADR 0636 §1).
|
||||||
|
// The finding writer is started first, so it holds what an earlier run left before the reader writes or clears any.
|
||||||
|
if session.Own == nil {
|
||||||
|
session.Own = link.NewOwnName()
|
||||||
|
}
|
||||||
|
findings := &lateFindings{own: session.Own, make: func(module string) (*findingsOfTheNode, error) {
|
||||||
|
return newFindingsOfTheNode(session, node, module, say)
|
||||||
|
}}
|
||||||
|
findings.Start(run, say)
|
||||||
|
seats, err := startSeatList(run, session, node, findings, say)
|
||||||
|
if err != nil {
|
||||||
|
// No reader is no gate that lets everything through: the node acts on nothing new, as one that waits does.
|
||||||
|
say(err.Error() + "; the node composes nothing new until it is started again and reads the list (novox/hq ADR 0636 §1)")
|
||||||
|
}
|
||||||
|
// The marks of the haves run off the composition pass, each bounded (see marker).
|
||||||
|
marks := newMarker(run)
|
||||||
|
heldFindings := &seatwait.HeldKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say}
|
||||||
|
// And what only this engine knows of itself, raised where it knows it (design 47 G A-197): its haves' failed phase, health
|
||||||
|
// and kept words as they are written, and each part of its health report at every look.
|
||||||
|
ownFindings := &seatwait.NodeKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say}
|
||||||
|
session.SetHaveSaid(ownFindings.Have, ownFindings.HaveGone)
|
||||||
|
ownFindingsNow.Store(ownFindings)
|
||||||
|
hooks := composerHooks{
|
||||||
|
entryNotFound: func(fp string, notFound bool) {
|
||||||
|
if err := seatwait.SayEntryNotFound(run, findings, node, findings.observer(), fp, notFound, time.Now); err != nil {
|
||||||
|
say("the finding that the catalog half holds no entry of " + fp + " cannot be kept: " + err.Error())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
seatList: seats,
|
||||||
|
seatListFailed: err != nil,
|
||||||
|
running: func() []compositions.Composed {
|
||||||
|
if now := own.get(); now != nil {
|
||||||
|
return runsAsWritten(now.Everything(), opts.state)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
unseal: sealOpener(opts.state),
|
||||||
|
// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
|
||||||
|
// A have the bus did not answer for is said as such, never as one holding no seals (issue 1126, 2026-10-28).
|
||||||
|
held: func(module string) (compositions.Seals, []string, error) {
|
||||||
|
if module == agent.NodeModule {
|
||||||
|
got, _, err := session.ReadNodeHaveErr(linked)
|
||||||
|
return got.SealedAt, got.Sealed, err
|
||||||
|
}
|
||||||
|
got, ok, err := session.ReadHaveErr(linked, module)
|
||||||
|
if !ok {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return got.SealedAt, got.Sealed, nil
|
||||||
|
},
|
||||||
|
refused: func(why string, held map[string]string) {
|
||||||
|
if now := own.get(); now != nil && run.Err() == nil {
|
||||||
|
// A module held alone marks that module, never the others (issue 1126); the marking never holds the pass.
|
||||||
|
marks.set("refused", func(ctx context.Context) { now.NotComposedEach(ctx, why, held) })
|
||||||
|
}
|
||||||
|
// And each module held says it as the finding `module-held`, cleared when it is composed again or no longer
|
||||||
|
// wanted here. A node refused whole (why) says nothing of its modules: their findings stand as they were.
|
||||||
|
if why == "" && run.Err() == nil {
|
||||||
|
marks.set("module-held", func(context.Context) { heldFindings.Reconcile(held) })
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// And what it keeps of a module as it runs, for a manifest that does not say it yet: on the module's have (ADR 0601 §4).
|
||||||
|
kept: func(kept map[string][]string) {
|
||||||
|
if now := own.get(); now != nil && run.Err() == nil {
|
||||||
|
marks.set("kept", func(ctx context.Context) { now.KeptNow(ctx, kept) })
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// And the vault's authority beside the controller's, and this node's certificate asked of the vault when a
|
||||||
|
// module here asks for one (ADR 0550 §2).
|
||||||
|
vaultAuthority: trust.VaultAuthority,
|
||||||
|
certify: func(ctx context.Context) error {
|
||||||
|
return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) })
|
||||||
|
},
|
||||||
|
}
|
||||||
|
// And its answer to every assignment candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||||
|
// its own key, `<node>.node`, under candidates.
|
||||||
|
shadowHooks := hooks
|
||||||
|
shadowHooks.refused, shadowHooks.kept, shadowHooks.certify, shadowHooks.seatList = nil, nil, nil, nil
|
||||||
|
shadowHooks.seatListFailed = false
|
||||||
|
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
||||||
|
shadow := selfcompose.NewShadow(shadowEnv,
|
||||||
|
func(ctx context.Context, answers map[string]nodeengine.CandidateAnswer) {
|
||||||
|
session.NoteCandidates(ctx, compositions.WithValuesDiffer(answers))
|
||||||
|
})
|
||||||
|
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
|
||||||
|
shadowNow.Store(shadow) // what the verb compose-candidate composes a check's candidate with
|
||||||
|
// Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's
|
||||||
|
// composition bucket) is never applied. While the seat's assignment holds no key for this node it composes
|
||||||
|
// nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again.
|
||||||
|
composer := selfComposer(session, node, hooks, say)
|
||||||
|
composer.AlsoEntryAgain = shadow.EntryAgain
|
||||||
|
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
|
||||||
|
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
|
||||||
|
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
||||||
|
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
|
||||||
|
go followRunning(run, session, composer, say)
|
||||||
|
go followBuildEnds(run, session, composer, say)
|
||||||
|
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
|
||||||
|
env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday
|
||||||
|
env.Recovery = recoveryEnv(session, composer)
|
||||||
|
env.WhyUnmade, env.Waiting, env.RunningManifest = composer.WhyUnmade, composer.Waiting, composer.RunningManifest
|
||||||
|
keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject)
|
||||||
|
keptFetch.Store(&keptFetchFn)
|
||||||
|
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||||
|
session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
|
||||||
ag = agent.New(env)
|
ag = agent.New(env)
|
||||||
|
own.set(ag)
|
||||||
current.set(ag)
|
current.set(ag)
|
||||||
err = ag.Run(linked)
|
// The node's own record is stated before the composer starts, from what the node knows of itself and with no bus (A-120).
|
||||||
|
sayFacts(run, nil)
|
||||||
|
err = ag.Run(run)
|
||||||
|
stopRun()
|
||||||
cancel()
|
cancel()
|
||||||
current.set(nil)
|
current.set(nil)
|
||||||
|
composerNow.Store(nil)
|
||||||
|
ownFindingsNow.Store(nil)
|
||||||
|
session.SetHaveSaid(nil, nil)
|
||||||
|
session.SetManifestOf(nil)
|
||||||
|
apply.UseEngineModule(nil)
|
||||||
|
shadowNow.Store(nil)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -502,14 +748,60 @@ var reloadSettles = 10 * time.Second
|
|||||||
type agentNow struct {
|
type agentNow struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
ag *agent.Agent
|
ag *agent.Agent
|
||||||
|
// last is the view of what runs the previous agent held when it stopped, kept in memory: while the next agent has
|
||||||
|
// read no haves yet, what runs is this and never the retired kept declaration (a bus link that drops restarts the
|
||||||
|
// agent, and every fill in that window was refused; issue 924).
|
||||||
|
last *runsView
|
||||||
|
// view reads an agent's view; viewOf unless a test stands in for an agent that holds the node.
|
||||||
|
view func(*agent.Agent) *runsView
|
||||||
|
}
|
||||||
|
|
||||||
|
func viewOf(a *agent.Agent) *runsView { return &runsView{a.Everything(), a.HoldsNode()} }
|
||||||
|
|
||||||
|
// runsView is what an agent said runs, to be read after the agent stopped.
|
||||||
|
type runsView struct {
|
||||||
|
everything []compositions.Composed
|
||||||
|
holdsNode bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *agentNow) set(a *agent.Agent) {
|
func (c *agentNow) set(a *agent.Agent) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
if a == nil && c.ag != nil {
|
||||||
|
read := c.view
|
||||||
|
if read == nil {
|
||||||
|
read = viewOf
|
||||||
|
}
|
||||||
|
if v := read(c.ag); len(v.everything) > 0 || v.holdsNode {
|
||||||
|
c.last = v
|
||||||
|
}
|
||||||
|
}
|
||||||
c.ag = a
|
c.ag = a
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *agentNow) viewFn() func(*agent.Agent) *runsView {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.view == nil {
|
||||||
|
return viewOf
|
||||||
|
}
|
||||||
|
return c.view
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdsNode drops the kept view once the agent in hand holds the node: its own view is complete from then on.
|
||||||
|
func (c *agentNow) holdsNode() {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.last = nil
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastView is the view the last agent held when it stopped, nil when none held any.
|
||||||
|
func (c *agentNow) lastView() *runsView {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.last
|
||||||
|
}
|
||||||
|
|
||||||
func (c *agentNow) get() *agent.Agent {
|
func (c *agentNow) get() *agent.Agent {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
@@ -517,7 +809,13 @@ func (c *agentNow) get() *agent.Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// applyComposition applies one composition: a module's, or the node's own.
|
// applyComposition applies one composition: a module's, or the node's own.
|
||||||
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed,
|
//
|
||||||
|
// placed is the modules a want still places here besides this one: none of their resources is removed by this apply,
|
||||||
|
// which is refused instead (issue 774).
|
||||||
|
//
|
||||||
|
// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when
|
||||||
|
// its module is among them (issue 774).
|
||||||
|
func applyComposition(ctx context.Context, opts options, c compositions.Composed, others []compositions.Composed, placed, removed map[string]bool,
|
||||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||||
name := scopeName(c)
|
name := scopeName(c)
|
||||||
if c.Module == "" {
|
if c.Module == "" {
|
||||||
@@ -533,17 +831,30 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
|
|||||||
if err := refuseEmptyWrites(d); err != nil {
|
if err := refuseEmptyWrites(d); err != nil {
|
||||||
return failedWith(err.Error())
|
return failedWith(err.Error())
|
||||||
}
|
}
|
||||||
scope := apply.Scope{Module: name, Others: resourcesOf(others)}
|
scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed}
|
||||||
|
if name == agent.NodeModule {
|
||||||
|
scope.RemovedByWant = removed
|
||||||
|
if scope.RemovedByWant == nil {
|
||||||
|
scope.RemovedByWant = map[string]bool{}
|
||||||
|
}
|
||||||
|
}
|
||||||
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
|
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
|
||||||
if got.Refused != "" {
|
if got.Refused != "" {
|
||||||
return failedWith(got.Refused)
|
return failedWith(got.Refused)
|
||||||
}
|
}
|
||||||
|
// An image that could not be pulled from a store is said with its holder, for the engine to try the apply again once the
|
||||||
|
// holder has settled (design 51, ENG-8).
|
||||||
|
failedOn := func(why string) agent.Result {
|
||||||
|
res := failedWith(why)
|
||||||
|
res.Pulls = pullsOf(d, got.Err)
|
||||||
|
return res
|
||||||
|
}
|
||||||
results := apply.ModuleResults(d, got.Outcome, got.Err)
|
results := apply.ModuleResults(d, got.Outcome, got.Err)
|
||||||
if r, ok := results[name]; ok && r.Failed {
|
if r, ok := results[name]; ok && r.Failed {
|
||||||
return failedWith(r.Detail)
|
return failedOn(r.Detail)
|
||||||
}
|
}
|
||||||
if got.Err != nil {
|
if got.Err != nil {
|
||||||
return failedWith(got.Err.Error())
|
return failedOn(got.Err.Error())
|
||||||
}
|
}
|
||||||
if got.Probe != nil {
|
if got.Probe != nil {
|
||||||
return failedWith(got.Probe.partial().Error())
|
return failedWith(got.Probe.partial().Error())
|
||||||
@@ -552,9 +863,9 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
|
|||||||
}
|
}
|
||||||
|
|
||||||
// removeComposition removes one module's own resources and nothing of another's.
|
// removeComposition removes one module's own resources and nothing of another's.
|
||||||
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed,
|
func removeComposition(ctx context.Context, opts options, module string, others []compositions.Composed, placed map[string]bool,
|
||||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||||
scope := apply.Scope{Module: module, Others: resourcesOf(others)}
|
scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed}
|
||||||
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
|
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
|
||||||
switch {
|
switch {
|
||||||
case got.Refused != "":
|
case got.Refused != "":
|
||||||
@@ -565,6 +876,30 @@ func removeComposition(ctx context.Context, opts options, module string, others
|
|||||||
return agent.Result{}
|
return agent.Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stopComposition stops a module's units and containers from the node's record and removes nothing (design 47 G A-98,
|
||||||
|
// §297): what the module kept stays, so a revert restores it before the reverted build starts.
|
||||||
|
func stopComposition(ctx context.Context, opts options, module string) agent.Result {
|
||||||
|
applying.Lock()
|
||||||
|
defer applying.Unlock()
|
||||||
|
unlock, err := store.Lock(opts.state, nil)
|
||||||
|
if err != nil {
|
||||||
|
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||||
|
}
|
||||||
|
built, err := system.For(builtFor)
|
||||||
|
if err != nil {
|
||||||
|
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||||
|
}
|
||||||
|
if _, err := apply.StopModule(ctx, built, known, module, opts.run()); err != nil {
|
||||||
|
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||||
|
}
|
||||||
|
return agent.Result{}
|
||||||
|
}
|
||||||
|
|
||||||
// busyModules are the modules whose last apply left a container running because it said busy.
|
// busyModules are the modules whose last apply left a container running because it said busy.
|
||||||
func busyModules() []string {
|
func busyModules() []string {
|
||||||
busyMu.Lock()
|
busyMu.Lock()
|
||||||
@@ -579,8 +914,35 @@ func busyModules() []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
|
// machinesNow is the composer whose assignments name the mesh's machines by address, for the health report's targets
|
||||||
|
// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target.
|
||||||
|
var machinesNow atomic.Pointer[selfcompose.Composer]
|
||||||
|
|
||||||
|
// ownFindingsNow is the keeper of the findings this engine raises of itself in the run that composes the node's own now, for the
|
||||||
|
// health loop (writeHealthReport); nil while no run holds one.
|
||||||
|
var ownFindingsNow atomic.Pointer[seatwait.NodeKeeper]
|
||||||
|
|
||||||
|
// composerNow is the composer of the run that composes the node's own now, nil while the node applies the controller's
|
||||||
|
// compositions or no run is going: what the verb compose asks a pass of (issue 774).
|
||||||
|
var composerNow atomic.Pointer[selfcompose.Composer]
|
||||||
|
|
||||||
|
// shadowNow is the shadow of the run now, which answers assignment candidates and the verb compose-candidate; nil
|
||||||
|
// between runs.
|
||||||
|
var shadowNow atomic.Pointer[selfcompose.Shadow]
|
||||||
|
|
||||||
|
// keptRetired says the kept declaration is read no more: every module the seat's assignment puts on this node has a have of
|
||||||
|
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
|
||||||
|
var keptRetired atomic.Bool
|
||||||
|
|
||||||
|
// errKeptRetired is keptDeclaration's answer once the read is retired.
|
||||||
|
var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own")
|
||||||
|
|
||||||
|
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given;
|
||||||
|
// once the read is retired, nothing is read.
|
||||||
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
|
if keptRetired.Load() {
|
||||||
|
return nil, errKeptRetired
|
||||||
|
}
|
||||||
var raw []byte
|
var raw []byte
|
||||||
if len(signer) > 0 {
|
if len(signer) > 0 {
|
||||||
var err error
|
var err error
|
||||||
@@ -602,7 +964,7 @@ func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.D
|
|||||||
// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept
|
// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept
|
||||||
// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of
|
// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of
|
||||||
// directories read.
|
// directories read.
|
||||||
func whatRuns(everything []delivery.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) {
|
func whatRuns(everything []compositions.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) {
|
||||||
d, err := declarationOf(everything...)
|
d, err := declarationOf(everything...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -651,11 +1013,23 @@ func declaredNow(statePath string, signer ed25519.PublicKey) (*declaration.Decla
|
|||||||
func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) }
|
kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) }
|
||||||
ag := current.get()
|
var view *runsView
|
||||||
if ag == nil {
|
if ag := current.get(); ag != nil {
|
||||||
|
view = current.viewFn()(ag)
|
||||||
|
if view.holdsNode {
|
||||||
|
current.holdsNode()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An agent that has read no haves yet, or none: the last view held, never the retired kept declaration.
|
||||||
|
if view == nil || (len(view.everything) == 0 && !view.holdsNode) {
|
||||||
|
if last := current.lastView(); last != nil {
|
||||||
|
view = last
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if view == nil {
|
||||||
return kept()
|
return kept()
|
||||||
}
|
}
|
||||||
return whatRuns(ag.Everything(), ag.HoldsNode(), kept)
|
return whatRuns(view.everything, view.holdsNode, kept)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -688,8 +1062,8 @@ func refuseEmptyWrites(d *declaration.Declaration) error {
|
|||||||
// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
|
// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
|
||||||
// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
|
// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
|
||||||
// that cannot be read says yes: the agent then believes what it was told, as before.
|
// that cannot be read says yes: the agent then believes what it was told, as before.
|
||||||
func recordedIn(statePath string, say func(string)) func(ctx context.Context, c delivery.Composed) bool {
|
func recordedIn(statePath string, say func(string)) func(ctx context.Context, c compositions.Composed) bool {
|
||||||
return func(_ context.Context, c delivery.Composed) bool {
|
return func(_ context.Context, c compositions.Composed) bool {
|
||||||
known, err := store.Load(statePath)
|
known, err := store.Load(statePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if say != nil {
|
if say != nil {
|
||||||
@@ -748,3 +1122,128 @@ const applyBound = 30 * time.Minute
|
|||||||
func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||||
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
|
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused).
|
||||||
|
type composerHooks struct {
|
||||||
|
running func() []compositions.Composed
|
||||||
|
unseal func(string) ([]byte, error)
|
||||||
|
refused func(why string, held map[string]string)
|
||||||
|
kept func(kept map[string][]string)
|
||||||
|
// entryNotFound is selfcompose.Env's EntryNotFound.
|
||||||
|
entryNotFound func(fp string, notFound bool)
|
||||||
|
held func(module string) (compositions.Seals, []string, error)
|
||||||
|
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
|
||||||
|
vaultAuthority func() string
|
||||||
|
certify func(ctx context.Context) error
|
||||||
|
// seatList is this run's reader of the seat list: the composer hands nothing new until it was read (ADR 0636 §1).
|
||||||
|
seatList *seatListState
|
||||||
|
// seatListFailed says the reader could not start: the composer then holds like a reader that waits (selfcompose.Unread).
|
||||||
|
seatListFailed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// selfComposer is the node's own composer: it composes what the seat's assignment places on the node, and nothing while
|
||||||
|
// the assignment holds no key for it (issue 1126).
|
||||||
|
func selfComposer(session *link.Session, node string, hooks composerHooks, say func(string)) *selfcompose.Composer {
|
||||||
|
env := composerEnv(session, node, say, hooks)
|
||||||
|
// And it keeps each placed module's data items in its own state data, which the node's disk-load watches (novox/hq
|
||||||
|
// ADR 0577 clause 1); the shadow composes for candidates and keeps none.
|
||||||
|
env.Data = link.EngineData{Session: session}
|
||||||
|
// And the packages its compositions declare in its own key of the state packages, which pacman reads (novox/hq ADR
|
||||||
|
// 0565 §6).
|
||||||
|
env.Packages = link.EnginePackages{Session: session}
|
||||||
|
// And every composition it makes in its composition state composed-<node>, a record the provisioner, the assigner
|
||||||
|
// and the view read and nothing applies (novox/hq ADR 0593 §6).
|
||||||
|
env.ComposedState = link.EngineComposed{Session: session, Node: node}
|
||||||
|
// And each composition it writes there said as the seat's event composed, which the assigner waits on.
|
||||||
|
env.SayComposed = session.SayComposed
|
||||||
|
return selfcompose.New(env)
|
||||||
|
}
|
||||||
|
|
||||||
|
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||||
|
// to assignment candidates (selfcompose.Shadow) alike.
|
||||||
|
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||||
|
env := selfcompose.Env{Node: node, NodeRecord: session.OwnRecord,
|
||||||
|
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||||
|
if bucket == buckets.Assignments || bucket == buckets.ProposalCandidates {
|
||||||
|
// The seat's states are made when their holder starts: a node-engine that starts first waits for the
|
||||||
|
// bus's word that the bucket was created, and the shadow and the composer say they wait.
|
||||||
|
return bucketWatch{session, bucket}.WatchAwait(ctx, pattern, func(err error) {
|
||||||
|
say("the seat's state " + bucket + " is not on the bus yet (" + err.Error() + "): it is watched when the bus says it was created")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||||
|
},
|
||||||
|
Get: func(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||||
|
h, err := session.KVHistory(ctx, bucket, key)
|
||||||
|
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return h[len(h)-1].Value, nil
|
||||||
|
},
|
||||||
|
Mesh: frombus.Mesh{InternalSuffix: overlay.Suffix()},
|
||||||
|
Say: say,
|
||||||
|
// Its modules' own secrets, sealed by the vault to this machine's key (ADR 0550 §5).
|
||||||
|
Own: func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
return session.AskVault(ctx, "own", args)
|
||||||
|
},
|
||||||
|
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||||
|
Hear: session.Hear,
|
||||||
|
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
||||||
|
Fetch: apply.Fetch,
|
||||||
|
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, EntryNotFound: hooks.entryNotFound,
|
||||||
|
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
|
||||||
|
if hooks.seatList != nil {
|
||||||
|
env.SeatList, env.SeatListChanged = hooks.seatList.reader, hooks.seatList.changed
|
||||||
|
} else if hooks.seatListFailed {
|
||||||
|
env.SeatList = selfcompose.Unread{}
|
||||||
|
}
|
||||||
|
// And the shadow too: the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's
|
||||||
|
// verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read.
|
||||||
|
env.Entry = selfcompose.EntryCall(session.AskMeshSeat)
|
||||||
|
// And the registry seats' verb versions, to resolve a want that names its version to its artifacts' digests (A-430a).
|
||||||
|
env.Registry = selfcompose.EntryCall(session.AskMeshSeat)
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
// followRunning keeps the composer told what every node runs, from the seat node-engine's event running, for the run it is
|
||||||
|
// in: the stream or the link not being there is said and tried again after a backoff, never polled.
|
||||||
|
func followRunning(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) {
|
||||||
|
keepFollowing(run, "this node does not hear what the nodes run, and tries again: ", func(ctx context.Context) error {
|
||||||
|
return session.FollowRunning(ctx, composer.NoteRunning, composer.RunningRead)
|
||||||
|
}, composer.RunningLost, say)
|
||||||
|
}
|
||||||
|
|
||||||
|
// followBuildEnds keeps the composer told how the builder's builds ended, from its event build.ended, for the run it is in
|
||||||
|
// (design 47 G A-293): a placed module's newest build that ended failed is said as a refusal in the composition state.
|
||||||
|
func followBuildEnds(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) {
|
||||||
|
keepFollowing(run, "this node does not hear how the builder's builds end, so a failed build is not said as refused, and it tries again: ",
|
||||||
|
func(ctx context.Context) error {
|
||||||
|
return session.FollowBuildEnds(ctx, composer.NoteBuildEnded, composer.BuildEndsRead)
|
||||||
|
}, composer.BuildEndsLost, say)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepFollowing follows a stream with follow for the run, lost called each time it ends; why it ended is said once until it
|
||||||
|
// changes, and it is followed again after a backoff, never polled.
|
||||||
|
func keepFollowing(run context.Context, line string, follow func(context.Context) error, lost func(), say func(string)) {
|
||||||
|
said := ""
|
||||||
|
for try := 1; run.Err() == nil; try++ {
|
||||||
|
began := time.Now()
|
||||||
|
err := follow(run)
|
||||||
|
lost()
|
||||||
|
if run.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if time.Since(began) > time.Minute {
|
||||||
|
try = 1
|
||||||
|
}
|
||||||
|
if why := fmt.Sprint(err); why != said {
|
||||||
|
say(line + why)
|
||||||
|
said = why
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-run.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(link.Backoff(try)):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -22,6 +22,11 @@ func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// The composition moved in from mesh-sdk (novox/hq ADR 0593 §7) has a node's input field of the same name,
|
||||||
|
// ReadsModules, which says the node reads a module per resource; it is no reconciling of a declaration.
|
||||||
|
if d.IsDir() && path == filepath.Join(root, "internal", "compose") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") {
|
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,12 +26,10 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
||||||
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of
|
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is.
|
||||||
// the user list on the link open now, told when the key is first heard or changes.
|
|
||||||
type heardKey struct {
|
type heardKey struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
pem string
|
pem string
|
||||||
kick func()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// vaultKey is the vault's key this engine heard.
|
// vaultKey is the vault's key this engine heard.
|
||||||
@@ -43,8 +41,7 @@ func (h *heardKey) get() string {
|
|||||||
return h.pem
|
return h.pem
|
||||||
}
|
}
|
||||||
|
|
||||||
// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer
|
// heard keeps the key a node-key answer gave. An answer with no key (a vault before this one) changes nothing.
|
||||||
// with no key (a vault before this one) changes nothing.
|
|
||||||
func (h *heardKey) heard(answer json.RawMessage) {
|
func (h *heardKey) heard(answer json.RawMessage) {
|
||||||
var a struct {
|
var a struct {
|
||||||
VaultKey string `json:"vault_key"`
|
VaultKey string `json:"vault_key"`
|
||||||
@@ -53,24 +50,8 @@ func (h *heardKey) heard(answer json.RawMessage) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
changed := a.VaultKey != h.pem
|
|
||||||
h.pem = a.VaultKey
|
h.pem = a.VaultKey
|
||||||
kick := h.kick
|
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
if changed && kick != nil {
|
|
||||||
kick()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// following names the follower told when the key changes, and tells it at once when a key is known.
|
|
||||||
func (h *heardKey) following(kick func()) {
|
|
||||||
h.mu.Lock()
|
|
||||||
h.kick = kick
|
|
||||||
known := h.pem != ""
|
|
||||||
h.mu.Unlock()
|
|
||||||
if known && kick != nil {
|
|
||||||
kick()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
||||||
|
|||||||
@@ -162,29 +162,22 @@ func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
||||||
// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one
|
// answer gives is kept; an answer without one (an older vault) or with no public key changes nothing (ADR 0547 clause
|
||||||
// (an older vault) or with no public key changes nothing (ADR 0547 clause 9).
|
// 9).
|
||||||
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
||||||
if link.NodeKeyContract != 13 {
|
if link.NodeKeyContract != 13 {
|
||||||
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
||||||
}
|
}
|
||||||
h := &heardKey{}
|
h := &heardKey{}
|
||||||
kicks := 0
|
|
||||||
h.following(func() { kicks++ })
|
|
||||||
h.heard(json.RawMessage(`{"recorded":true}`))
|
h.heard(json.RawMessage(`{"recorded":true}`))
|
||||||
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
||||||
if h.get() != "" || kicks != 0 {
|
if h.get() != "" {
|
||||||
t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks)
|
t.Fatalf("an answer with no key was kept: %q", h.get())
|
||||||
}
|
}
|
||||||
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
||||||
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
||||||
h.heard(answer)
|
h.heard(answer)
|
||||||
h.heard(answer)
|
if h.get() != pem {
|
||||||
if h.get() != pem || kicks != 1 {
|
t.Fatalf("key %q, want the key", h.get())
|
||||||
t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks)
|
|
||||||
}
|
|
||||||
h.following(func() { kicks += 10 })
|
|
||||||
if kicks != 11 {
|
|
||||||
t.Fatalf("a follower named after the key was heard was not told: %d", kicks)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Design 47 G §527, §634, A-149: the node-engine's own module is the module of its applied record at its own fingerprint,
|
||||||
|
// and its finding writer is made, observing as that module, once the name is known: at once from the record, else when the
|
||||||
|
// first composing names it. No name is fixed in code.
|
||||||
|
func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testing.T) {
|
||||||
|
fp := "src2:" + strings.Repeat("a", 64)
|
||||||
|
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
|
||||||
|
for module, f := range map[string]string{"node": "", "the-engine": fp, "other": "src2:" + strings.Repeat("b", 64)} {
|
||||||
|
if err := applied.Keep(module, []byte(`{"module":"`+module+`","fp":"`+f+`"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, tc := range []struct{ fp, want string }{{fp, "the-engine"}, {"", ""}, {"src2:" + strings.Repeat("c", 64), ""}} {
|
||||||
|
if got := ownModuleApplied(applied, tc.fp); got != tc.want {
|
||||||
|
t.Errorf("the own module at %q is %q, want %q", tc.fp, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
own := link.NewOwnName()
|
||||||
|
made := make(chan string, 1)
|
||||||
|
late := &lateFindings{own: own, make: func(module string) (*findingsOfTheNode, error) {
|
||||||
|
made <- module
|
||||||
|
return nil, context.Canceled // the writer itself is the SDK's: only when it is made is tested here
|
||||||
|
}}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
var said []string
|
||||||
|
late.Start(ctx, func(s string) { said = append(said, s) })
|
||||||
|
if err := late.Write(ctx, condseat.Finding{}); err == nil || late.Held() != nil {
|
||||||
|
t.Fatalf("a finding was taken before the engine's own module was known: %v", err)
|
||||||
|
}
|
||||||
|
if len(made) != 0 {
|
||||||
|
t.Fatal("the writer was made before a name")
|
||||||
|
}
|
||||||
|
own.Set("the-engine") // the first composing names it
|
||||||
|
select {
|
||||||
|
case m := <-made:
|
||||||
|
if m != "the-engine" || late.observer() != "the-engine" {
|
||||||
|
t.Fatalf("the writer was made as %q, observing as %q", m, late.observer())
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the writer was never made after the name was known")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// §633: the engine's bound for the first seat list is the file the manifest's `view-waits-minutes` feeds, five minutes
|
||||||
|
// when it holds nothing usable.
|
||||||
|
func TestTheViewWaitsBoundIsTheManifestsSetting(t *testing.T) {
|
||||||
|
was := viewWaitsFile
|
||||||
|
defer func() { viewWaitsFile = was }()
|
||||||
|
for content, want := range map[string]time.Duration{"2\n": 2 * time.Minute, "": 5 * time.Minute, "0": 5 * time.Minute, "x": 5 * time.Minute} {
|
||||||
|
viewWaitsFile = filepath.Join(t.TempDir(), "w")
|
||||||
|
if err := os.WriteFile(viewWaitsFile, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := viewWaitsBound(); got != want {
|
||||||
|
t.Errorf("the file %q gives %v, want %v", content, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
viewWaitsFile = filepath.Join(t.TempDir(), "missing")
|
||||||
|
if got := viewWaitsBound(); got != 5*time.Minute {
|
||||||
|
t.Errorf("no file gives %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -88,11 +88,12 @@ func TestThePackagesGoInWhenTheApplyThatRestoresThePackageManagerWroteIt(t *test
|
|||||||
conf := filepath.Join(dir, "pacman.conf")
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
pacmanAnsweringOnce(t, conf)
|
pacmanAnsweringOnce(t, conf)
|
||||||
opts := options{state: filepath.Join(dir, "state.json")}
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
// The package first: within one pass it is asked about before the file is written, which is the order that
|
// The file first, as the pacman module declares it: a module's apply stops at its first failed resource
|
||||||
// leaves it to a second pass.
|
// (design 42 zfs-owns-zfs §1, answer A-26), so the file that makes the package manager answer comes before the
|
||||||
|
// package that needs it.
|
||||||
raw := []byte(`{"declaration":1,"resources":[` +
|
raw := []byte(`{"declaration":1,"resources":[` +
|
||||||
`{"id":"pacman.base","type":"package","package":"base"},` +
|
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"}]}`)
|
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||||
|
|
||||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write
|
||||||
|
// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:<provision>}`, and the node-engine
|
||||||
|
// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the
|
||||||
|
// placeholder alone.
|
||||||
|
//
|
||||||
|
// A placeholder names the credential as the module calls it, `${secret:<name>}` (catalogue.SecretLocal, ADR 0094): the
|
||||||
|
// local name where the module keeps several values of one provision, else the provision. For each, it finds the one
|
||||||
|
// provision key `<node>.<consumer>.<provision>[.<local>]` of the seat mesh-provisioner's state `provisions` (clause 1)
|
||||||
|
// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair
|
||||||
|
// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client,
|
||||||
|
// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the
|
||||||
|
// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot
|
||||||
|
// connect.
|
||||||
|
type provisionSecrets struct {
|
||||||
|
node string
|
||||||
|
// list is every key of mesh-provisioner's `provisions` under prefix (`<node>.<consumer>.`), by key: what this
|
||||||
|
// engine may read, its own node's (`<node>.>`).
|
||||||
|
list func(ctx context.Context, prefix string) (map[string][]byte, error)
|
||||||
|
// own asks the vault's verb `own` in this engine's name.
|
||||||
|
own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
|
||||||
|
// open opens a value sealed to this machine.
|
||||||
|
open func(sealed string) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretPlaceholder is `${secret:<name>}`.
|
||||||
|
var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`)
|
||||||
|
|
||||||
|
// fill is content with each provision placeholder of module's filled, and whether any was.
|
||||||
|
func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) {
|
||||||
|
found := secretPlaceholder.FindAllSubmatch(content, -1)
|
||||||
|
if len(found) == 0 {
|
||||||
|
return content, false, nil
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found))
|
||||||
|
}
|
||||||
|
if p.list == nil || p.own == nil || p.open == nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module)
|
||||||
|
}
|
||||||
|
prefix := p.node + "." + module + "."
|
||||||
|
keys, err := p.list(ctx, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err)
|
||||||
|
}
|
||||||
|
out := string(content)
|
||||||
|
done := map[string]bool{}
|
||||||
|
for _, f := range found {
|
||||||
|
whole, name := string(f[0]), string(f[1])
|
||||||
|
if done[whole] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
done[whole] = true
|
||||||
|
pr, key, err := byName(keys, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err)
|
||||||
|
}
|
||||||
|
value, err := p.value(ctx, key, pr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
out = strings.ReplaceAll(out, whole, string(value))
|
||||||
|
}
|
||||||
|
return []byte(out), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key.
|
||||||
|
func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) {
|
||||||
|
var hit []string
|
||||||
|
var got provisions.Provision
|
||||||
|
for key, raw := range keys {
|
||||||
|
pr, err := provisions.Decode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if catalogue.SecretLocal(pr.Provision, pr.Local) == name {
|
||||||
|
hit, got = append(hit, key), pr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(hit) {
|
||||||
|
case 0:
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", buckets.ProvisionsRead)
|
||||||
|
case 1:
|
||||||
|
return got, hit[0], nil
|
||||||
|
}
|
||||||
|
sort.Strings(hit)
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// value is one provision's value on this node, opened.
|
||||||
|
func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) {
|
||||||
|
if pr.State != provisions.Provided || pr.Reference == nil {
|
||||||
|
why := pr.State
|
||||||
|
if pr.Why != "" {
|
||||||
|
why += ": " + pr.Why
|
||||||
|
}
|
||||||
|
if pr.State == provisions.Provided {
|
||||||
|
why = "provided with no reference to the vault's value"
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why)
|
||||||
|
}
|
||||||
|
// The reference names a credential of this provision, as the vault names it (provisions.VaultName).
|
||||||
|
ours := false
|
||||||
|
for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} {
|
||||||
|
ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID
|
||||||
|
}
|
||||||
|
if !ours {
|
||||||
|
return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID)
|
||||||
|
}
|
||||||
|
module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local
|
||||||
|
type asked struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Provision string `json:"provision"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
ConsumerModule string `json:"consumer_module"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
Local string `json:"local,omitempty"`
|
||||||
|
}
|
||||||
|
ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local}
|
||||||
|
answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s: %w", key, err)
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Credentials []struct {
|
||||||
|
asked
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
|
} `json:"credentials"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer, &a); err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err)
|
||||||
|
}
|
||||||
|
if a.Node != p.node {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node)
|
||||||
|
}
|
||||||
|
for _, c := range a.Credentials {
|
||||||
|
if c.asked != ask {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The value the provision names, by the vault's credential name and its generation (clause 6): another is refused.
|
||||||
|
if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+
|
||||||
|
"nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation)
|
||||||
|
}
|
||||||
|
value, err := p.open(string(c.Sealed))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the vault answered no value of %s for this machine", key)
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pullsOf is the images an apply could not find or pull, each with the seat it came through and that seat's holder as the
|
||||||
|
// declaration names them beside the container (design 51 *distribution's first delivery*, ENG-8, ENG-10). An image the
|
||||||
|
// declaration names no holder for (a public registry's) carries none: its pull is not tried again on a store settling.
|
||||||
|
func pullsOf(d *declaration.Declaration, err error) []agent.PullFailure {
|
||||||
|
var out []agent.PullFailure
|
||||||
|
for _, p := range apply.PullErrors(err) {
|
||||||
|
f := agent.PullFailure{Image: p.Image, Started: p.Started}
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if c, ok := r.(*declaration.Container); ok && c.Image == p.Image && c.ImageHolder != "" {
|
||||||
|
f.Seat, f.Holder = c.ImageSeat, c.ImageHolder
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -3,11 +3,10 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/liveness"
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
@@ -21,7 +20,7 @@ func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
|||||||
if err := store.Save(path, s); err != nil {
|
if err := store.Save(path, s); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
|
c := engcomp.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
|
||||||
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
|
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
|
||||||
recorded := recordedIn(path, func(string) {})
|
recorded := recordedIn(path, func(string) {})
|
||||||
if !recorded(context.Background(), c) {
|
if !recorded(context.Background(), c) {
|
||||||
@@ -36,6 +35,7 @@ func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
|||||||
|
|
||||||
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
||||||
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||||
|
t.Run("1957: the machine shows a resource not active", testTheMachineShowsAResourceNotActive)
|
||||||
st := liveness.Statement{Resources: []liveness.State{
|
st := liveness.Statement{Resources: []liveness.State{
|
||||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
||||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
||||||
@@ -51,6 +51,7 @@ func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
|||||||
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
||||||
// that comes back and goes again is newly missing again (the review of issue 1945).
|
// that comes back and goes again is newly missing again (the review of issue 1945).
|
||||||
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
||||||
|
t.Run("1957: a resource newly down is an edge", testOnlyAResourceNewlyDownIsAnEdge)
|
||||||
missing := func(ids ...string) liveness.Statement {
|
missing := func(ids ...string) liveness.Statement {
|
||||||
var st liveness.Statement
|
var st liveness.Statement
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
@@ -85,7 +86,7 @@ func TestAFilledFileDoesNotMakeItsCompositionUnrecorded(t *testing.T) {
|
|||||||
if err := store.Save(path, s); err != nil {
|
if err := store.Save(path, s); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
c := delivery.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{
|
c := engcomp.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{
|
||||||
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`),
|
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`),
|
||||||
json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}}
|
json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}}
|
||||||
if !recordedIn(path, func(string) {})(context.Background(), c) {
|
if !recordedIn(path, func(string) {})(context.Background(), c) {
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296,
|
||||||
|
// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the
|
||||||
|
// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool.
|
||||||
|
func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery {
|
||||||
|
ask := func(ctx context.Context, verb string, in, out any) error {
|
||||||
|
raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(raw, out)
|
||||||
|
}
|
||||||
|
return &agent.Recovery{
|
||||||
|
FormChanges: composer.FormChanges,
|
||||||
|
SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a)
|
||||||
|
},
|
||||||
|
RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a)
|
||||||
|
},
|
||||||
|
KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a)
|
||||||
|
},
|
||||||
|
Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) {
|
||||||
|
heard, stop, err := session.HearRecoveryEnding(kind, module, kept)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
out := make(chan agent.Ending, 64)
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case ev := <-heard:
|
||||||
|
select {
|
||||||
|
case out <- agent.Ending(ev):
|
||||||
|
case <-done:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-done:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return out, func() { stop(); close(done) }, nil
|
||||||
|
},
|
||||||
|
Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) {
|
||||||
|
return session.KeepBound(ctx, what, id, module, since, d)
|
||||||
|
},
|
||||||
|
CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error {
|
||||||
|
return session.CallTool(ctx, module, c.Tool, c.Arguments)
|
||||||
|
},
|
||||||
|
WaitMinutes: recoveryWaitMinutes(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the
|
||||||
|
// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71).
|
||||||
|
var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes"
|
||||||
|
|
||||||
|
// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the
|
||||||
|
// file is missing, empty or not a positive number (design 47 G §554, A-112).
|
||||||
|
func recoveryWaitMinutes() int {
|
||||||
|
raw, err := os.ReadFile(recoveryWaitFile)
|
||||||
|
if err != nil {
|
||||||
|
return agent.RecoveryWaitMinutes
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return agent.RecoveryWaitMinutes
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
@@ -0,0 +1,301 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||||
|
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/nodegrant"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears
|
||||||
|
// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's
|
||||||
|
// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and
|
||||||
|
// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant.
|
||||||
|
func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) {
|
||||||
|
perms := nodegrant.For(itNode)
|
||||||
|
perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode,
|
||||||
|
"mesh.mod.db.tool.rebuild."+itNode)
|
||||||
|
perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>")
|
||||||
|
b := newItBedAs(t, perms)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll},
|
||||||
|
{meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} {
|
||||||
|
if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
js, err := jetstream.New(b.conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
say := func(act, outcome string) {
|
||||||
|
raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()})
|
||||||
|
if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var restoreAsked meshrecovery.RestoreKeptInput
|
||||||
|
// The kept points the stand-in answers: the one of the first checks, then (below) what the agent's revert finds, and none.
|
||||||
|
var pointsMu sync.Mutex
|
||||||
|
points := []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1",
|
||||||
|
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}
|
||||||
|
// The restore the agent's revert asks waits for the stand-in to say it restored: gated until released.
|
||||||
|
var gated atomic.Bool
|
||||||
|
release := make(chan struct{})
|
||||||
|
var restores atomic.Int32
|
||||||
|
for verb, answer := range map[string]func(m *nats.Msg) any{
|
||||||
|
meshrecovery.VerbKeptPoints: func(*nats.Msg) any {
|
||||||
|
pointsMu.Lock()
|
||||||
|
defer pointsMu.Unlock()
|
||||||
|
return meshrecovery.KeptPointsAnswer{Points: points}
|
||||||
|
},
|
||||||
|
meshrecovery.VerbRestoreKept: func(m *nats.Msg) any {
|
||||||
|
pointsMu.Lock()
|
||||||
|
_ = json.Unmarshal(m.Data, &restoreAsked)
|
||||||
|
pointsMu.Unlock()
|
||||||
|
act := fmt.Sprintf("r%d", restores.Add(1))
|
||||||
|
if gated.Load() {
|
||||||
|
go func() { <-release; say(act, meshrecovery.RestoredEventOutcomeDone) }()
|
||||||
|
} else {
|
||||||
|
say(act, meshrecovery.RestoredEventOutcomeDone)
|
||||||
|
}
|
||||||
|
return meshrecovery.RestoreKeptAnswer{Accepted: act}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
answer := answer
|
||||||
|
sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) {
|
||||||
|
out, _ := json.Marshal(map[string]any{"result": answer(m)})
|
||||||
|
_ = m.Respond(out)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
}
|
||||||
|
var toolArgs map[string]any
|
||||||
|
var toolCalls atomic.Int32
|
||||||
|
toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) {
|
||||||
|
var args map[string]any
|
||||||
|
_ = json.Unmarshal(m.Data, &args)
|
||||||
|
pointsMu.Lock()
|
||||||
|
toolArgs = args
|
||||||
|
pointsMu.Unlock()
|
||||||
|
toolCalls.Add(1)
|
||||||
|
_ = m.Respond([]byte(`{"result":{}}`))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = toolSub.Unsubscribe() })
|
||||||
|
|
||||||
|
// A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number.
|
||||||
|
for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} {
|
||||||
|
recoveryWaitFile = filepath.Join(t.TempDir(), "w")
|
||||||
|
if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recoveryWaitMinutes(); got != want {
|
||||||
|
t.Errorf("the file %q gives %d minutes, want %d", content, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
recoveryWaitFile = filepath.Join(t.TempDir(), "missing")
|
||||||
|
if got := recoveryWaitMinutes(); got != 90 {
|
||||||
|
t.Errorf("no file gives %d minutes, want 90", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject
|
||||||
|
session := b.startSession()
|
||||||
|
r := recoveryEnv(session, nil)
|
||||||
|
|
||||||
|
events, stop, err := r.Hear(ctx, "restored", "db", "cache")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
if first := <-events; first.Act != "old" || first.Outcome != "failed" {
|
||||||
|
t.Fatalf("the first event heard is %+v, want the last one said before", first)
|
||||||
|
}
|
||||||
|
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"})
|
||||||
|
if err != nil || len(pts.Points) != 1 {
|
||||||
|
t.Fatalf("kept-points: %+v, %v", pts, err)
|
||||||
|
}
|
||||||
|
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"})
|
||||||
|
pointsMu.Lock()
|
||||||
|
firstAsked := restoreAsked
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if err != nil || got.Accepted != "r1" || firstAsked.Snapshot != "snap-1" {
|
||||||
|
t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, firstAsked)
|
||||||
|
}
|
||||||
|
if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" {
|
||||||
|
t.Fatalf("the ending heard is %+v", ev)
|
||||||
|
}
|
||||||
|
err = r.CatchUp(ctx, "db", *pts.Points[0].CatchUp)
|
||||||
|
pointsMu.Lock()
|
||||||
|
gotArgs := toolArgs
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if err != nil || gotArgs["all"] != true {
|
||||||
|
t.Fatalf("catch-up: %v, arguments %v", err, gotArgs)
|
||||||
|
}
|
||||||
|
expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer end()
|
||||||
|
select {
|
||||||
|
case <-expired:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("the bound's end, the bus's marker, was not heard")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Design 47 G §296, §297, §554, §560, A-113: the node agent whole drives a revert want of db through the same recovery wiring
|
||||||
|
// (recoveryEnv over the session and the node's composer): db runs at a build whose cache is of form 2, and its want goes back to a
|
||||||
|
// build of form 1. With no point of form 1 taken before the newer build, its have says the restore point is missing, naming it,
|
||||||
|
// and the newer build runs again. With the point, the have is held (backing up) until the stand-in says restored, then the
|
||||||
|
// older build is applied and the point's catch-up tool is called.
|
||||||
|
dbManifest := func(form int, content string) string {
|
||||||
|
return `{"module":"db","version":"1","state":[{"name":"cache","form":` + fmt.Sprint(form) + `}],"resources":[{"id":"conf","type":"file","path":"` +
|
||||||
|
filepath.Join(b.dir, "db.conf") + `","mode":"0644","content":"` + content + `\n"}]}`
|
||||||
|
}
|
||||||
|
fpOld, fpNew := fpIt("a"), fpIt("b")
|
||||||
|
b.registerEntry("db", fpOld, dbManifest(1, "v1"))
|
||||||
|
b.registerEntry("db", fpNew, dbManifest(2, "v2"))
|
||||||
|
for _, name := range []string{buckets.Manifests, selfcompose.ResolvedBucket, buckets.ProposalCandidates, link.PackagesBucket} {
|
||||||
|
if _, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 5}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assignment, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: buckets.Assignments, History: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "db"}}})
|
||||||
|
hkey, _ := nec.HaveNodeKey(itNode)
|
||||||
|
rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}})
|
||||||
|
if _, err := b.haves.Put(hkey, rec); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for key, value := range places {
|
||||||
|
if _, err := assignment.Put(key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.want("db", fpNew, "", "")
|
||||||
|
stopAgent := b.start()
|
||||||
|
defer stopAgent()
|
||||||
|
within(t, 20*time.Second, "db does not run its newer build", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Fp == fpNew && h.Phase == nec.PhaseRunning && b.file("db") == "v2\n"
|
||||||
|
})
|
||||||
|
// The engine says what runs, as it does after each apply: the composer keeps the entry of the running build while the want moves on.
|
||||||
|
b.running(itNode, "db", fpNew, meshcatalogclient.ManifestDigest([]byte(dbManifest(2, "v2"))))
|
||||||
|
// The stand-in holds no point of form 1: the revert is held, the have names the point missing, the newer build runs.
|
||||||
|
pointsMu.Lock()
|
||||||
|
points = nil
|
||||||
|
pointsMu.Unlock()
|
||||||
|
b.want("db", fpOld, "", "")
|
||||||
|
within(t, 20*time.Second, "the have does not say the restore point is missing", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Phase == nec.PhaseHeld && strings.Contains(h.Detail, "restore point missing") &&
|
||||||
|
strings.Contains(h.Detail, "db's cache has none of form 1 taken before "+fpNew)
|
||||||
|
})
|
||||||
|
if b.have("db").Fp != fpNew || b.file("db") != "v2\n" || restores.Load() != 1 {
|
||||||
|
t.Fatalf("the missing point left db %+v with %q, %d restore(s) asked", b.have("db"), b.file("db"), restores.Load())
|
||||||
|
}
|
||||||
|
// The stand-in now holds the point: the want answered "again" asks the restore, and the have waits for its ending.
|
||||||
|
pointsMu.Lock()
|
||||||
|
points = []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fpNew, Snapshot: "snap-2", At: time.Now(),
|
||||||
|
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": "again"}}}}
|
||||||
|
pointsMu.Unlock()
|
||||||
|
gated.Store(true)
|
||||||
|
calls := toolCalls.Load()
|
||||||
|
b.want("db", fpOld, "", meshwants.WantsByWantNodeModuleValueBackupAgain)
|
||||||
|
within(t, 20*time.Second, "the restore of the revert is not asked", func() bool { return restores.Load() == 2 })
|
||||||
|
within(t, 20*time.Second, "the have does not wait for the restore", func() bool { return b.have("db").Phase == nec.PhaseBackingUp })
|
||||||
|
// The restored event is held until close(release): the older build must not be applied before the stand-in says restored.
|
||||||
|
if h := b.have("db"); h.Phase != nec.PhaseBackingUp || b.file("db") != "v2\n" || toolCalls.Load() != calls {
|
||||||
|
t.Fatalf("before the stand-in says restored the have is %+v, the file %q, %d catch-up call(s) more", h, b.file("db"), toolCalls.Load()-calls)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
within(t, 20*time.Second, "the older build is not applied after the restore", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Fp == fpOld && h.Phase == nec.PhaseRunning && b.file("db") == "v1\n"
|
||||||
|
})
|
||||||
|
within(t, 20*time.Second, "the catch-up tool is not called after the restore", func() bool { return toolCalls.Load() == calls+1 })
|
||||||
|
pointsMu.Lock()
|
||||||
|
asked, gotArgs := restoreAsked, toolArgs
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if asked.Snapshot != "snap-2" || asked.Fingerprint != fpNew || gotArgs["all"] != "again" {
|
||||||
|
t.Errorf("the revert asked %+v and called the catch-up with %v", asked, gotArgs)
|
||||||
|
}
|
||||||
|
stopAgent()
|
||||||
|
|
||||||
|
// Design 47 G A-293: over the same session, the builder's build ends the composer follows: the stream read whole first (the
|
||||||
|
// last end of each module and fingerprint, in the stream's order), then the read's end, then each end said after it.
|
||||||
|
if _, err := b.js.AddStream(&nats.StreamConfig{Name: link.BuildEndsStream, Subjects: []string{link.BuildEndsSubject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ended := func(fp, outcome, why string) {
|
||||||
|
raw, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: fp, Outcome: outcome, Error: why})
|
||||||
|
if _, err := b.js.Publish("mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("db", fp), raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ended("f1", builds.BuildFailed, "first")
|
||||||
|
ended("f1", builds.BuildFailed, "no go")
|
||||||
|
ended("f0", builds.BuildDone, "")
|
||||||
|
heard := make(chan string, 8)
|
||||||
|
follow, stopFollow := context.WithCancel(ctx)
|
||||||
|
defer stopFollow()
|
||||||
|
go func() {
|
||||||
|
_ = session.FollowBuildEnds(follow, func(e builds.BuildEnded) { heard <- e.Fingerprint + ":" + e.Outcome + ":" + e.Error },
|
||||||
|
func() { heard <- "read" })
|
||||||
|
}()
|
||||||
|
for i, want := range []string{"f1:failed:no go", "f0:done:", "read", "f2:failed:later"} {
|
||||||
|
if want == "f2:failed:later" {
|
||||||
|
ended("f2", builds.BuildFailed, "later")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case got := <-heard:
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("build end %d heard %q, want %q", i, got, want)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("build end %d, %q, was not heard", i, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSession is a session on the node's own connection, with no node agent run on it.
|
||||||
|
func (b *itBed) startSession() *link.Session {
|
||||||
|
b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||||
|
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||||
|
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||||
|
return b.session
|
||||||
|
}
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/ed25519"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/tunnel"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
|
|
||||||
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
|
|
||||||
// proof signed by the identity key, over the previous key, the new one and the tunnel.
|
|
||||||
|
|
||||||
func anEnrolledNode(t *testing.T) identity.Identity {
|
|
||||||
t.Helper()
|
|
||||||
mine, err := identity.Generate("anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Overlay, err = identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
|
|
||||||
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
|
|
||||||
return mine
|
|
||||||
}
|
|
||||||
|
|
||||||
func aFoundTunnel(t *testing.T) tunnel.Found {
|
|
||||||
t.Helper()
|
|
||||||
private, err := identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
|
|
||||||
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
|
|
||||||
mine := anEnrolledNode(t)
|
|
||||||
found := aFoundTunnel(t)
|
|
||||||
before := mine.Overlay.Public
|
|
||||||
|
|
||||||
taken, rekey, err := rekeyOnto(mine, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
|
|
||||||
t.Fatal("the overlay key is not the tunnel's")
|
|
||||||
}
|
|
||||||
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
|
|
||||||
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
|
|
||||||
taken.Membership.Broker != mine.Membership.Broker {
|
|
||||||
t.Fatal("something other than the overlay key moved")
|
|
||||||
}
|
|
||||||
if taken.OverlayBefore != before {
|
|
||||||
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
|
|
||||||
}
|
|
||||||
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
|
|
||||||
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
|
|
||||||
t.Fatalf("the rekey does not say what moved: %+v", rekey)
|
|
||||||
}
|
|
||||||
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the rekey is not signed by this node's identity key over what it says")
|
|
||||||
}
|
|
||||||
if ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the proof is not bound to the node")
|
|
||||||
}
|
|
||||||
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
|
|
||||||
if strings.Contains(said, found.PrivateKey()) {
|
|
||||||
t.Fatal("the private key travels")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
|
|
||||||
// names is still the one before the take, so the mesh can tell a repeat from a replay.
|
|
||||||
again, second, err := rekeyOnto(taken, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
|
|
||||||
t.Fatalf("a take run again does not name the key before the first: %+v", second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest
|
||||||
|
// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the
|
||||||
|
// agent has not applied since its start can come from the kept declaration, older than the compositions applied since:
|
||||||
|
// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the
|
||||||
|
// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it
|
||||||
|
// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote,
|
||||||
|
// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is
|
||||||
|
// left as the composition names it.
|
||||||
|
func runsAsWritten(running []engcomp.Composed, statePath string) []engcomp.Composed {
|
||||||
|
if len(running) == 0 || statePath == "" {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
out := make([]engcomp.Composed, len(running))
|
||||||
|
for i, c := range running {
|
||||||
|
out[i] = c
|
||||||
|
var changed []json.RawMessage
|
||||||
|
for j, raw := range c.Resources {
|
||||||
|
if fixed, ok := asWritten(raw, known); ok {
|
||||||
|
if changed == nil {
|
||||||
|
changed = append([]json.RawMessage(nil), c.Resources...)
|
||||||
|
}
|
||||||
|
changed[j] = fixed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if changed != nil {
|
||||||
|
out[i].Resources = changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a
|
||||||
|
// file given by another digest and the file there is what the record says the node wrote.
|
||||||
|
func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) {
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(raw, &fields) != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
var kind, id, path, digest string
|
||||||
|
_ = json.Unmarshal(fields["type"], &kind)
|
||||||
|
_ = json.Unmarshal(fields["id"], &id)
|
||||||
|
_ = json.Unmarshal(fields["path"], &path)
|
||||||
|
_ = json.Unmarshal(fields["digest"], &digest)
|
||||||
|
if kind != string(declaration.TypeFile) || digest == "" || path == "" {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
rec, ok := known.Find(id)
|
||||||
|
if !ok || rec.Wrote == "" || rec.Target != path {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(sum[:]) != rec.Wrote {
|
||||||
|
return nil, false // changed since the node wrote it: not the node's to vouch for
|
||||||
|
}
|
||||||
|
if now := declaration.DigestOf(body); now == digest {
|
||||||
|
return nil, false
|
||||||
|
} else {
|
||||||
|
fields["digest"], _ = json.Marshal(now)
|
||||||
|
}
|
||||||
|
fields["size"], _ = json.Marshal(len(body))
|
||||||
|
fixed, err := json.Marshal(fields)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return fixed, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/views"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/seatwait"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-engine reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636): through the seat mesh-catalog,
|
||||||
|
// from the bootstrapped definitions until it has been read, acting on nothing new meanwhile (the composer's gate), and
|
||||||
|
// following the seat's events. What it holds is given to catalogue at every change, which compiles no seat list.
|
||||||
|
|
||||||
|
// seatListNow is this run's reader, what the node-engine's verbs seat-list and seat-list-again answer from.
|
||||||
|
var seatListNow atomic.Pointer[seatListState]
|
||||||
|
|
||||||
|
// seatListRead answers the verb seat-list: what the engine holds of the list and what it waits for (ADR 0636 §7).
|
||||||
|
func seatListRead() (any, error) {
|
||||||
|
st := seatListNow.Load()
|
||||||
|
if st == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
||||||
|
}
|
||||||
|
status := st.reader.Status()
|
||||||
|
return map[string]any{"status": status, "health": status.Health()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatListAgain answers the verb seat-list-again: the list read now from the seat, and what is held afterwards.
|
||||||
|
func seatListAgain() (any, error) {
|
||||||
|
st := seatListNow.Load()
|
||||||
|
if st == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
||||||
|
}
|
||||||
|
err := st.reader.Reload()
|
||||||
|
st.waiter.Changed() // a read that changed nothing held still ends the wait
|
||||||
|
out, _ := seatListRead()
|
||||||
|
if err != nil {
|
||||||
|
out.(map[string]any)["error"] = err.Error()
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatListState is one run's reader, the finding that it waits, and the channel on which the composer hears a change.
|
||||||
|
type seatListState struct {
|
||||||
|
reader *seatlist.Reader
|
||||||
|
waiter *seatwait.Waiter
|
||||||
|
changed chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sessionBus is the reader's bus over the link: a verb of the seat is asked in the engine's own name, an event heard on
|
||||||
|
// the seat's event subject for as long as ctx lives.
|
||||||
|
type sessionBus struct {
|
||||||
|
ctx context.Context
|
||||||
|
session *link.Session
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b sessionBus) Ask(key string, body any) (json.RawMessage, error) {
|
||||||
|
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||||
|
return b.session.AskMeshSeat(b.ctx, seat, verb, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b sessionBus) Subscribe(pattern string, handle func(string, json.RawMessage) error) error {
|
||||||
|
seat, event, _ := strings.Cut(pattern, ".")
|
||||||
|
heard, stop, err := b.session.Hear(seatdef.EventSubject(seat, event))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-b.ctx.Done():
|
||||||
|
return
|
||||||
|
case body, ok := <-heard:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = handle(pattern, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatWaitClock is the clock of the node's bound; the tests give their own.
|
||||||
|
var seatWaitClock seatwait.Clock = realClock{}
|
||||||
|
|
||||||
|
type realClock struct{}
|
||||||
|
|
||||||
|
func (realClock) Now() time.Time { return time.Now() }
|
||||||
|
func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer {
|
||||||
|
return time.AfterFunc(d, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they
|
||||||
|
// happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions.
|
||||||
|
type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
|
||||||
|
|
||||||
|
// findingsOfTheNode keeps the node's findings through mesh-sdk's held writer (design 47 G §383, §384, §461, A-29): each
|
||||||
|
// finding is written in the node's state `findings` and said as an event on the engine's own stream (a foundation part's
|
||||||
|
// `mesh-host-findings`, §529, A-30), the engine says what it holds at its start and when the keeper asks, and the keeper
|
||||||
|
// reads no state of the engine's.
|
||||||
|
type findingsOfTheNode struct {
|
||||||
|
*nodeFindings
|
||||||
|
}
|
||||||
|
|
||||||
|
// newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on
|
||||||
|
// the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own.
|
||||||
|
func newFindingsOfTheNode(session *link.Session, node, module string, say func(string)) (*findingsOfTheNode, error) {
|
||||||
|
writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
|
||||||
|
Module: module,
|
||||||
|
Node: node,
|
||||||
|
State: link.FindingsState{Session: session},
|
||||||
|
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...),
|
||||||
|
Asked: condseat.SubjectFindingsAsked,
|
||||||
|
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
|
||||||
|
Publish: session.PublishEvent,
|
||||||
|
Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) {
|
||||||
|
return session.ListenFrom(condseat.StreamFindingsAsked, subject, handle)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &findingsOfTheNode{writer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start seeds what the state holds, states the findings now and hears the keeper until ctx ends.
|
||||||
|
func (w *findingsOfTheNode) Start(ctx context.Context) error {
|
||||||
|
go func() { <-ctx.Done(); w.Close() }()
|
||||||
|
return w.nodeFindings.Start(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what
|
||||||
|
// it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what
|
||||||
|
// the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds.
|
||||||
|
func startSeatList(ctx context.Context, session *link.Session, node string, findings *lateFindings, say func(string)) (*seatListState, error) {
|
||||||
|
st := &seatListState{changed: make(chan struct{}, 1)}
|
||||||
|
// Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists.
|
||||||
|
var announce atomic.Pointer[func()]
|
||||||
|
reader, err := seatlist.Start(seatlist.Config{Bus: sessionBus{ctx, session}, Bootstrap: seatlist.Bootstrapped(),
|
||||||
|
OnChange: func(seats []catalogue.Seat) {
|
||||||
|
catalogue.UseSeats(seats)
|
||||||
|
// What the engine reads changed with the list: its record and its event say it (ADR 0618 §4), off the
|
||||||
|
// reader's goroutine, which returns quickly.
|
||||||
|
go session.RestateReads(ctx)
|
||||||
|
if f := announce.Load(); f != nil { // the first read may end before the waiter exists
|
||||||
|
(*f)()
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err)
|
||||||
|
}
|
||||||
|
st.reader = reader
|
||||||
|
st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Bound: viewWaitsBound(),
|
||||||
|
Clock: seatWaitClock, Say: say}
|
||||||
|
told := func() {
|
||||||
|
st.waiter.Changed()
|
||||||
|
select {
|
||||||
|
case st.changed <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
announce.Store(&told)
|
||||||
|
catalogue.UseSeats(reader.Seats())
|
||||||
|
session.RestateReads(ctx) // its first statement holds the bootstrapped seats at least (issue 1966)
|
||||||
|
told()
|
||||||
|
// The first read ends whether the seat answered or not, and a read that held what the reader held already says no
|
||||||
|
// change: its end is told to the waiter and the composer, so a status that moved from reading to heard starts or ends
|
||||||
|
// the bound (ADR 0636 §1). Start's return held nobody's start.
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-reader.Ready():
|
||||||
|
catalogue.UseSeats(reader.Seats())
|
||||||
|
session.RestateReads(ctx)
|
||||||
|
told()
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
seatListNow.Store(st)
|
||||||
|
return st, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lateFindings is the node-engine's finding writer, made when the engine's own module is known (design 47 G §527, §634,
|
||||||
|
// A-149): at once from its applied record, or, on a first start with no such record, when the first composing names it. Till
|
||||||
|
// then a write or a clear is refused in words, never lost in silence.
|
||||||
|
type lateFindings struct {
|
||||||
|
own *link.OwnName
|
||||||
|
make func(module string) (*findingsOfTheNode, error)
|
||||||
|
mu sync.Mutex
|
||||||
|
w *findingsOfTheNode
|
||||||
|
}
|
||||||
|
|
||||||
|
// observer is the module the findings are observed as: the engine's own, "" while it is not known.
|
||||||
|
func (f *lateFindings) observer() string { return f.own.Get() }
|
||||||
|
|
||||||
|
// Start makes the writer and starts it as soon as the own name is known: in this call when it already is, else on a goroutine.
|
||||||
|
func (f *lateFindings) Start(ctx context.Context, say func(string)) {
|
||||||
|
begin := func(module string) {
|
||||||
|
w, err := f.make(module)
|
||||||
|
if err != nil { // a writer that cannot be made is a fault of its configuration: loud
|
||||||
|
say("the node-engine cannot make its finding writer: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := w.Start(ctx); err != nil {
|
||||||
|
say("the node-engine's findings are kept in its state and not said as events until it is started again: " + err.Error())
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.w = w
|
||||||
|
f.mu.Unlock()
|
||||||
|
}
|
||||||
|
if name := f.own.Get(); name != "" {
|
||||||
|
begin(name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if name, err := f.own.Wait(ctx); err == nil {
|
||||||
|
begin(name)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
var errOwnNameUnknown = errors.New("the node-engine's own module is not yet named by its applied record or its first composing, so its finding writer is not made")
|
||||||
|
|
||||||
|
func (f *lateFindings) writer() *findingsOfTheNode {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.w
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Write(ctx context.Context, fi condseat.Finding) error {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Write(ctx, fi)
|
||||||
|
}
|
||||||
|
return errOwnNameUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Clear(ctx context.Context, scope, id, kind, why string) error {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Clear(ctx, scope, id, kind, why)
|
||||||
|
}
|
||||||
|
return errOwnNameUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Held() []condseat.Finding {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Held()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// viewWaitsFile is the file the manifest declares with the content ${config:view-waits-minutes} (design 47 G §633), read at
|
||||||
|
// the engine's start as recoveryWaitFile is.
|
||||||
|
var viewWaitsFile = "/var/lib/mesh-host/view-waits-minutes"
|
||||||
|
|
||||||
|
// viewWaitsBound is how long the reader of the seat list may wait before the engine says it: the file's minutes, the design's
|
||||||
|
// five (seatwait.Bound) when the file is missing, empty or not a positive number.
|
||||||
|
func viewWaitsBound() time.Duration {
|
||||||
|
raw, err := os.ReadFile(viewWaitsFile)
|
||||||
|
if err != nil {
|
||||||
|
return seatwait.Bound
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return seatwait.Bound
|
||||||
|
}
|
||||||
|
return time.Duration(n) * time.Minute
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ADR 0636 §1: when the node's reader of the seat list could not start, the composer is gated by a reader that never
|
||||||
|
// allows, where a missing reader would let the node act on everything; the shadow, which applies nothing, stays ungated.
|
||||||
|
func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) {
|
||||||
|
failed := composerEnv(nil, "n1", func(string) {}, composerHooks{seatListFailed: true})
|
||||||
|
if failed.SeatList == nil || failed.SeatList.MayAct() || failed.SeatList.MayAct("node-engine") {
|
||||||
|
t.Fatalf("the composer of a node whose reader did not start is gated by %v, want a gate that allows nothing", failed.SeatList)
|
||||||
|
}
|
||||||
|
if _, ok := failed.SeatList.(selfcompose.Unread); !ok {
|
||||||
|
t.Errorf("gate is %T", failed.SeatList)
|
||||||
|
}
|
||||||
|
if got := composerEnv(nil, "n1", func(string) {}, composerHooks{}).SeatList; got != nil {
|
||||||
|
t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// settledEvent is where the status view says a module has settled on a node: its have reached its want, running and healthy
|
||||||
|
// (design 47; the view's `settled`, said once for a module on a node at one composition). The engine hears it for every module
|
||||||
|
// and tells its agent, which tries again, once per settled, a failed apply that waits on that module as the holder of the store
|
||||||
|
// its image comes from (design 51 *distribution's first delivery*, issue 1966 ENG-11). The engine's manifest consumes it (`mesh-delivery-view.settled`),
|
||||||
|
// and that line alone grants the engine's account the subject (TestTheSettledEventIsTheManifestsConsume); a refused subscription is said, and made again when the next link opens.
|
||||||
|
const settledEvent = "mesh.mod.mesh-delivery-view.event.settled"
|
||||||
|
|
||||||
|
// settledBody is what of the event the engine reads: the module that settled, on whichever node.
|
||||||
|
type settledBody struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledTold is who is told a holder has settled.
|
||||||
|
type settledTold interface{ Settled(holder string) }
|
||||||
|
|
||||||
|
// followSettled hears the status view's `settled` on every link, never on a timer, until ctx ends. now names the agent running
|
||||||
|
// now (nil while the node agent begins again): a settled heard while there is none is not lost for it, since a new agent applies
|
||||||
|
// again every have that failed at its start.
|
||||||
|
func followSettled(ctx context.Context, session *link.Session, now func() settledTold, say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
events, stop, herr := session.Hear(settledEvent)
|
||||||
|
if herr != nil {
|
||||||
|
if line := fmt.Sprintf("a store settling is not heard yet (%s: %v): a failed pull waits for the next link, or for compose", settledEvent, herr); line != said {
|
||||||
|
said = line
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
said = ""
|
||||||
|
listen(linked, events, now, say)
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
<-linked.Done()
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// listen tells the agent each settled it hears, until the link ends.
|
||||||
|
func listen(linked context.Context, events <-chan []byte, now func() settledTold, say link.Announce) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
return
|
||||||
|
case raw, ok := <-events:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var b settledBody
|
||||||
|
if json.Unmarshal(raw, &b) != nil || b.Module == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if told := now(); told != nil {
|
||||||
|
told.Settled(b.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The subject the engine hears for a store settling is the event its manifest consumes: the grants part grants an account
|
||||||
|
// what its manifest consumes and nothing named in code (novox/hq ADR 0625 §4).
|
||||||
|
func TestTheSettledEventIsTheManifestsConsume(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
emitter, event, _ := strings.Cut(strings.TrimPrefix(settledEvent, "mesh.mod."), ".event.")
|
||||||
|
if !slices.Contains(m.Consumes, emitter+"."+event) {
|
||||||
|
t.Errorf("the engine hears %s, its manifest consumes %v", settledEvent, m.Consumes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The manifest claims the seat node-engine, says its findings on its own stream and hears the keeper's ask and the
|
||||||
|
// node-engine seat's running (design 47 G §364, §387, §199).
|
||||||
|
func TestTheManifestClaimsItsSeatAndSaysItsFindings(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
Claims []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
} `json:"claims"`
|
||||||
|
Streams []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Events []string `json:"events"`
|
||||||
|
} `json:"streams"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(m.Claims) != 1 || m.Claims[0].Name != "node-engine" || m.Claims[0].Scope != "node" {
|
||||||
|
t.Errorf("claims = %+v, want the node seat node-engine", m.Claims)
|
||||||
|
}
|
||||||
|
if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-host-findings" ||
|
||||||
|
!slices.Equal(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
|
||||||
|
t.Errorf("streams = %+v, want mesh-host-findings with the three finding events", m.Streams)
|
||||||
|
}
|
||||||
|
for _, c := range []string{"mesh-conditions.findings-asked", "node-engine.running"} {
|
||||||
|
if !slices.Contains(m.Consumes, c) {
|
||||||
|
t.Errorf("consumes %v lacks %s", m.Consumes, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.Version != "0.1" {
|
||||||
|
t.Errorf("version = %q, want MAJOR.MINOR 0.1 (§368, §411)", m.Version)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the
|
||||||
|
// record, so the module's kept data and its record survive (design 47 G A-98).
|
||||||
|
func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) {
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
var s store.State
|
||||||
|
s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"})
|
||||||
|
s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"})
|
||||||
|
if err := store.Save(path, s); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var asked []string
|
||||||
|
opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}}
|
||||||
|
if res := stopComposition(context.Background(), opts, "app"); res.Failed {
|
||||||
|
t.Fatalf("stop failed: %s", res.Detail)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") ||
|
||||||
|
strings.Contains(strings.Join(asked, "\n"), " rm ") {
|
||||||
|
t.Fatalf("the machine was asked %v", asked)
|
||||||
|
}
|
||||||
|
if after, err := store.Load(path); err != nil || len(after.Resources) != 2 {
|
||||||
|
t.Fatalf("the record changed: %v, %v", after.Resources, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stopBound is how long a stopping engine waits for the apply in flight to end and save what it did: under the service
|
||||||
|
// manager's own stop bound (90 s by default, then every process of the unit is killed), so the wait ends with a line in
|
||||||
|
// the journal rather than with a kill.
|
||||||
|
const stopBound = 60 * time.Second
|
||||||
|
|
||||||
|
// stopBetweenApplies is how this engine ends, whichever way (a stop of the service, standing aside for a successor, a
|
||||||
|
// move to another bus): it waits, up to bound, for the apply in flight to end, then lets no save start and waits for
|
||||||
|
// the one writing (novox/hq issue 1585). An apply runs detached from the engine's own stop (applyContext), so without
|
||||||
|
// this the process ended under it: what it unpacked was never recorded, and a save cut between its write and its
|
||||||
|
// rename left a temporary file and the state before it. A stopping engine never gives back what it takes (the process
|
||||||
|
// ends); the release it returns is for a test, and gives back the apply only when this took it.
|
||||||
|
func stopBetweenApplies(say func(string), bound time.Duration) (release func()) {
|
||||||
|
took := true
|
||||||
|
if !applying.TryLock() {
|
||||||
|
say(fmt.Sprintf("stopping: waiting up to %s for the apply in flight to end and save what it did", bound))
|
||||||
|
got := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
applying.Lock()
|
||||||
|
close(got)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-got:
|
||||||
|
say("stopping: the apply in flight ended and its state is saved")
|
||||||
|
case <-time.After(bound):
|
||||||
|
took = false
|
||||||
|
say(fmt.Sprintf("stopping while an apply still runs after %s: what it did since its last save is not "+
|
||||||
|
"recorded, and the next start applies its module again", bound))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saves := store.Quiesce()
|
||||||
|
return func() {
|
||||||
|
saves()
|
||||||
|
if took {
|
||||||
|
applying.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeLeftoverSaves removes, at the engine's start, the temporary files of saves of the node's state that never
|
||||||
|
// reached their rename, and says how many (issue 1585: about 260 on novox, ~80 MB).
|
||||||
|
func removeLeftoverSaves(statePath string, say func(string)) {
|
||||||
|
removed, err := store.RemoveLeftovers(statePath)
|
||||||
|
if len(removed) > 0 {
|
||||||
|
shown := removed
|
||||||
|
if len(shown) > 5 {
|
||||||
|
shown = append(append([]string(nil), shown[:5]...), "…")
|
||||||
|
}
|
||||||
|
say(fmt.Sprintf("removed %d leftover save(s) of this node's state, cut off before their rename by an earlier "+
|
||||||
|
"stop: %s", len(removed), strings.Join(shown, ", ")))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say(err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// exitBetweenApplies ends the process cleanly once no apply runs and no save writes: an exit that is not runLink's
|
||||||
|
// return (a move to another bus). A variable, so a test sees it called rather than the test process ending.
|
||||||
|
var exitBetweenApplies = func(say func(string)) {
|
||||||
|
_ = stopBetweenApplies(say, stopBound)
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d; issue 1126): the
|
||||||
|
// vault's key alone, read from the vault's state `signers`, which only the vault may write: the bus grants are its
|
||||||
|
// anchor, as the wants' and the assignment's. The controller's key, which the node was enrolled with (its membership's
|
||||||
|
// signer), is trusted never: the controller ended by its unassignment (the operator, 2026-11-02) and its lease is never
|
||||||
|
// handed, so no gate waits on it. What it trusts now the node states in its record (NodeRecord.Trusts).
|
||||||
|
type meshTrust struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
vault ed25519.PublicKey
|
||||||
|
authority string
|
||||||
|
// changed is told when what it trusts changed, so the record is written again.
|
||||||
|
changed func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMeshTrust trusts nothing until the vault's key is read.
|
||||||
|
func newMeshTrust() *meshTrust { return &meshTrust{} }
|
||||||
|
|
||||||
|
// onChange names who is told when what it trusts changed.
|
||||||
|
func (t *meshTrust) onChange(f func()) {
|
||||||
|
t.mu.Lock()
|
||||||
|
t.changed = f
|
||||||
|
t.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signers are the keys a mesh signature is verified against now: the vault's once read, and none before.
|
||||||
|
func (t *meshTrust) Signers() []meshvaultclient.Signer {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
var out []meshvaultclient.Signer
|
||||||
|
if t.vault != nil {
|
||||||
|
out = append(out, meshvaultclient.Signer{Name: meshvaultclient.SignerVault, Key: t.vault})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify is the signer that signed msg as kind, or why none it trusts did.
|
||||||
|
func (t *meshTrust) Verify(kind string, msg, sig []byte) (meshvaultclient.Signer, error) {
|
||||||
|
context, err := meshvaultclient.SignContext(kind)
|
||||||
|
if err != nil {
|
||||||
|
return meshvaultclient.Signer{}, err
|
||||||
|
}
|
||||||
|
return meshvaultclient.VerifyBy(t.Signers(), context, msg, sig)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
|
||||||
|
func (t *meshTrust) VaultAuthority() string {
|
||||||
|
if t == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.authority
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
|
||||||
|
func (t *meshTrust) Trusts() []string {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
authorities := map[string]string{}
|
||||||
|
if a := t.VaultAuthority(); a != "" {
|
||||||
|
authorities[meshvaultclient.SignerVault] = a
|
||||||
|
}
|
||||||
|
return meshvaultclient.Trusts(t.Signers(), authorities)
|
||||||
|
}
|
||||||
|
|
||||||
|
// change applies f under the lock and tells who follows when Trusts differs after it.
|
||||||
|
func (t *meshTrust) change(f func()) {
|
||||||
|
before := strings.Join(t.Trusts(), "\n")
|
||||||
|
t.mu.Lock()
|
||||||
|
f()
|
||||||
|
changed := t.changed
|
||||||
|
t.mu.Unlock()
|
||||||
|
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
|
||||||
|
changed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
|
||||||
|
// check; a deleted key is no longer trusted.
|
||||||
|
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
|
||||||
|
switch key {
|
||||||
|
case meshvault.SignersByKeyKey():
|
||||||
|
var pub ed25519.PublicKey
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
k, err := meshvaultclient.DecodeSignerKey(value)
|
||||||
|
if err == nil {
|
||||||
|
pub, err = meshvaultclient.SignerPublicKey(k)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's signing key in " + buckets.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.change(func() { t.vault = pub })
|
||||||
|
case meshvault.SignersByAuthorityKey():
|
||||||
|
pem := ""
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
a, err := meshvaultclient.DecodeSignerAuthority(value)
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's authority in " + buckets.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pem = a.Certificate
|
||||||
|
}
|
||||||
|
t.change(func() { t.authority = pem })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
|
||||||
|
// not serve `sign` yet, and until it does this node is not granted the reads.
|
||||||
|
var trustFollowAgain = 2 * time.Minute
|
||||||
|
|
||||||
|
// followTrust follows the vault's `signers` on every link, as watches (never a poll), for as
|
||||||
|
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
|
||||||
|
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
sayOnce := func(line string) {
|
||||||
|
if line != said {
|
||||||
|
said = line
|
||||||
|
if line != "" {
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
watching, stop := context.WithCancel(linked)
|
||||||
|
signers, serr := session.KVWatch(watching, buckets.Signers, ">")
|
||||||
|
if serr != nil {
|
||||||
|
stop()
|
||||||
|
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v): "+
|
||||||
|
"it trusts what it last read, and looks again in %s", serr, trustFollowAgain))
|
||||||
|
} else {
|
||||||
|
sayOnce("")
|
||||||
|
for u := range signers {
|
||||||
|
if !u.Caught {
|
||||||
|
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
case <-time.After(trustFollowAgain):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedAllHave says whether every module the seat mesh-module-assigner places on node has a have of its own, read from the
|
||||||
|
// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose assignment or haves
|
||||||
|
// cannot be read, keeps reading it.
|
||||||
|
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
|
||||||
|
key, err := meshmoduleassignerclient.NodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
// Its node key names every module placed (research 072 row 37).
|
||||||
|
values := map[string][]byte{}
|
||||||
|
if h, err := session.KVHistory(ctx, buckets.Assignments, key); err == nil && len(h) > 0 && !h[len(h)-1].Deleted {
|
||||||
|
values[key] = h[len(h)-1].Value
|
||||||
|
}
|
||||||
|
p, err := placedModules(node, values)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
return allHave(p, func(module string) (nodeengine.HaveByNodeModuleValue, bool, error) {
|
||||||
|
return session.ReadHaveErr(ctx, module)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedModules is the modules node's node key names (an assignment of their names alone); an error when it does not
|
||||||
|
// read.
|
||||||
|
func placedModules(node string, values map[string][]byte) (meshmoduleassigner.Assignments, error) {
|
||||||
|
key, _ := meshmoduleassignerclient.NodeKey(node)
|
||||||
|
raw, ok := values[key]
|
||||||
|
if !ok {
|
||||||
|
return meshmoduleassigner.Assignments{}, errors.New("its assignment is not read")
|
||||||
|
}
|
||||||
|
n, err := meshmoduleassignerclient.DecodeNode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return meshmoduleassigner.Assignments{}, err
|
||||||
|
}
|
||||||
|
p := meshmoduleassigner.Assignments{Node: node}
|
||||||
|
for _, r := range n.Modules {
|
||||||
|
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: r.Module})
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// allHave says whether has finds, for every module p places, a have that names a fingerprint of the new form, and why
|
||||||
|
// not. A module whose have names only a build of the old form runs inside the node's own composition, and the kept
|
||||||
|
// declaration is the one record of what runs of it: retired then, the node's own composed it as nothing and refused to
|
||||||
|
// remove what runs (issue 774, 2026-10-30 19:43: systemd-resolved on shanks, nats and restic on novox, each node's own
|
||||||
|
// apply refused from then on).
|
||||||
|
func allHave(p meshmoduleassigner.Assignments, has func(module string) (nodeengine.HaveByNodeModuleValue, bool, error)) (bool, string) {
|
||||||
|
if len(p.Modules) == 0 {
|
||||||
|
return false, "the seat mesh-module-assigner places no module here"
|
||||||
|
}
|
||||||
|
for _, m := range p.Modules {
|
||||||
|
h, ok, err := has(m.Module)
|
||||||
|
if err != nil {
|
||||||
|
return false, "the have of " + m.Module + " is not read: " + err.Error()
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return false, m.Module + " has no have yet"
|
||||||
|
}
|
||||||
|
// A module the mesh provides (mesh-wireguard) never has a fingerprint: the node's own composes it from the manifest
|
||||||
|
// mesh-sdk provides (overlay.ProvidedManifest), never from the kept declaration, so it keeps no read.
|
||||||
|
if h.Phase != nodeengineclient.PhaseRemoved && !strings.HasPrefix(h.Fp, meshcatalogclient.SourcePrefix2) && !overlay.ProvidedByMesh(m.Module) {
|
||||||
|
return false, m.Module + " runs at a build of the old form (" + h.Build + "), which only the kept declaration records"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
|
||||||
|
}
|
||||||
|
|
||||||
|
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
|
||||||
|
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
|
||||||
|
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
|
||||||
|
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
|
||||||
|
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
|
||||||
|
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key, err := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||||
|
return fmt.Errorf("this node's serving key is not an ed25519 key")
|
||||||
|
}
|
||||||
|
raw, err := ask(ctx, meshvault.VerbSign, map[string]any{"kind": meshvaultclient.KindNodeCertificate, "node": node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var a meshvault.SignAnswer
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return fmt.Errorf("the vault's answer is not a signing: %w", err)
|
||||||
|
}
|
||||||
|
trusted := trust.VaultAuthority()
|
||||||
|
if trusted == "" {
|
||||||
|
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
|
||||||
|
}
|
||||||
|
if err := meshvaultclient.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
|
||||||
|
ed25519.PublicKey(key), time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
path := identity.CertificatePath(state)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if written != nil {
|
||||||
|
written()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -9,6 +12,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/accounts"
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/liveness"
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/units"
|
"github.com/novox/mesh-host/internal/units"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
|||||||
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
||||||
t.Fatalf("root said: %v", got)
|
t.Fatalf("root said: %v", got)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old
|
||||||
|
// with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why.
|
||||||
|
t.Run("this machine's root-free", func(t *testing.T) {
|
||||||
|
agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
|
||||||
|
State: accounts.Healthy, Since: at}
|
||||||
|
free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
st *accounts.Statement
|
||||||
|
after time.Duration
|
||||||
|
served bool
|
||||||
|
free bool
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"},
|
||||||
|
{name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"},
|
||||||
|
{name: "no verdict", st: nil, why: "judged no account yet"},
|
||||||
|
{name: "a way to root", st: as, why: "in the group docker, which grants root"},
|
||||||
|
{name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"},
|
||||||
|
{name: "execute served", st: free, served: true, why: "the login shell runs any command"},
|
||||||
|
} {
|
||||||
|
v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here")
|
||||||
|
if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" {
|
||||||
|
t.Errorf("%s: %+v", row.name, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// execute is served while the holder's setting file says anything but withhold, or the bus hears it.
|
||||||
|
dir := t.TempDir()
|
||||||
|
statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf")
|
||||||
|
if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
quiet := func() (bool, error) { return false, nil }
|
||||||
|
for _, row := range []struct {
|
||||||
|
name, setting string
|
||||||
|
heard func() (bool, error)
|
||||||
|
served bool
|
||||||
|
}{
|
||||||
|
{name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet},
|
||||||
|
{name: "served", setting: "execute=serve\n", heard: quiet, served: true},
|
||||||
|
{name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true},
|
||||||
|
{name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true},
|
||||||
|
{name: "no setting file", heard: quiet, served: true},
|
||||||
|
} {
|
||||||
|
_ = os.Remove(conf)
|
||||||
|
if row.setting != "" {
|
||||||
|
if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served, why := executeServedHere(statePath, row.heard); served != row.served {
|
||||||
|
t.Errorf("%s: served %v (%s)", row.name, served, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
||||||
|
|||||||
+85
-29
@@ -2,36 +2,36 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
"github.com/novox/mesh-host/internal/userlist"
|
"github.com/novox/mesh-host/internal/userlist"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the bus's user list alone from the newest
|
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the grants part's user list alone into the file
|
||||||
// composition of the bus module, under the apply's two locks as a filled file is written (filling.go), and says what
|
// the bus module's composition declares, under the apply's two locks as a filled file is written (filling.go), and says what
|
||||||
// it wrote in the bus module's have.
|
// it wrote in the bus module's have.
|
||||||
|
|
||||||
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
||||||
// agent trusts, the write alone, and the have.
|
// agent trusts, the write alone, and the have.
|
||||||
func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey,
|
func userListFollower(opts options, session *link.Session, running func() []engcomp.Composed,
|
||||||
say link.Announce, wrote func()) *userlist.Follower {
|
say link.Announce, wrote func()) *userlist.Follower {
|
||||||
return userlist.New(userlist.Env{
|
return userlist.New(userlist.Env{
|
||||||
Node: session.Membership.Node,
|
Node: session.Membership.Node,
|
||||||
Composed: bucketWatch{session, composedBucket},
|
Running: running,
|
||||||
Trusted: trusted,
|
OnDisk: userListOnDisk,
|
||||||
Resolve: func(ctx context.Context, raw json.RawMessage) ([]byte, error) {
|
|
||||||
return resolveUserList(session.WithLink(ctx), raw)
|
|
||||||
},
|
|
||||||
OnDisk: userListOnDisk,
|
|
||||||
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
||||||
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
||||||
if changed && wrote != nil {
|
if changed && wrote != nil {
|
||||||
@@ -42,13 +42,42 @@ func userListFollower(opts options, session *link.Session, composedBucket string
|
|||||||
Report: func(ctx context.Context, module string, w userlist.Written) {
|
Report: func(ctx context.Context, module string, w userlist.Written) {
|
||||||
session.NoteUserList(ctx, module, w)
|
session.NoteUserList(ctx, module, w)
|
||||||
},
|
},
|
||||||
// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
|
// From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and
|
||||||
// clause 9).
|
// opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it.
|
||||||
VaultKey: vaultKey.get,
|
Grants: bucketWatch{session, grantsstate.GrantsUserListBucket},
|
||||||
Say: announceOr(say),
|
Fetch: func(ctx context.Context, name string) ([]byte, error) {
|
||||||
|
return session.GrantsObject(ctx, name, grantsListLimit)
|
||||||
|
},
|
||||||
|
Open: func(b seal.Box) ([]byte, error) {
|
||||||
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return key.OpenVaultBox(b)
|
||||||
|
},
|
||||||
|
Key: func() string {
|
||||||
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
pem, err := key.PublicPEM()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return pem
|
||||||
|
},
|
||||||
|
// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
|
||||||
|
// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
|
||||||
|
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
|
||||||
|
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
|
||||||
|
Backoff: link.Backoff,
|
||||||
|
Say: announceOr(say),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes.
|
||||||
|
const grantsListLimit = 16 << 20
|
||||||
|
|
||||||
// userListFile is a user-list resource as the file it declares.
|
// userListFile is a user-list resource as the file it declares.
|
||||||
func userListFile(raw []byte) (*declaration.File, error) {
|
func userListFile(raw []byte) (*declaration.File, error) {
|
||||||
r, err := declaration.ParseResource(raw)
|
r, err := declaration.ParseResource(raw)
|
||||||
@@ -62,20 +91,6 @@ func userListFile(raw []byte) (*declaration.File, error) {
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveUserList is the list a user-list resource holds: its content, or the object its digest names in this node's
|
|
||||||
// file bucket, fetched and verified (apply.ResolveFileReferences, ADR 0486 §1).
|
|
||||||
func resolveUserList(ctx context.Context, raw []byte) ([]byte, error) {
|
|
||||||
f, err := userListFile(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
d := &declaration.Declaration{Version: declaration.Version, Resources: []declaration.Resource{f}}
|
|
||||||
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
|
||||||
return nil, fmt.Errorf("the user list could not be had by its digest: %w", err)
|
|
||||||
}
|
|
||||||
return []byte(d.Resources[0].(*declaration.File).Content), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// userListOnDisk is what the user list's file holds on this machine now; nil when there is none.
|
// userListOnDisk is what the user list's file holds on this machine now; nil when there is none.
|
||||||
func userListOnDisk(raw json.RawMessage) ([]byte, error) {
|
func userListOnDisk(raw json.RawMessage) ([]byte, error) {
|
||||||
f, err := userListFile(raw)
|
f, err := userListFile(raw)
|
||||||
@@ -125,3 +140,44 @@ func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []by
|
|||||||
}
|
}
|
||||||
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
|
||||||
|
func grantsKeyPath(opts options) string {
|
||||||
|
return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
type keptKey struct {
|
||||||
|
Revision uint64 `json:"revision"`
|
||||||
|
Value json.RawMessage `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func keptGrantsKey(opts options) ([]byte, uint64) {
|
||||||
|
raw, err := os.ReadFile(grantsKeyPath(opts))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0
|
||||||
|
}
|
||||||
|
var k keptKey
|
||||||
|
if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
|
||||||
|
return nil, 0
|
||||||
|
}
|
||||||
|
return k.Value, k.Revision
|
||||||
|
}
|
||||||
|
|
||||||
|
func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
|
||||||
|
path := grantsKeyPath(opts)
|
||||||
|
if value == nil {
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
|
||||||
|
tmp := path + ".new"
|
||||||
|
err := os.WriteFile(tmp, raw, 0o600)
|
||||||
|
if err == nil {
|
||||||
|
err = os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+178
-12
@@ -1,16 +1,28 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/agent"
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func aFileOf(module, id string) json.RawMessage {
|
func aFileOf(module, id string) json.RawMessage {
|
||||||
@@ -25,7 +37,7 @@ func aFileOf(module, id string) json.RawMessage {
|
|||||||
func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
||||||
kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}},
|
kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}},
|
||||||
Libraries: []declaration.Library{{Library: "media"}}}
|
Libraries: []declaration.Library{{Library: "media"}}}
|
||||||
everything := []delivery.Composed{
|
everything := []engcomp.Composed{
|
||||||
{Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}},
|
{Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}},
|
||||||
{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's
|
{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's
|
||||||
}
|
}
|
||||||
@@ -39,7 +51,7 @@ func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Once the agent holds the node's composition, its parts are the node's.
|
// Once the agent holds the node's composition, its parts are the node's.
|
||||||
node := delivery.Composed{Node: "n1", Own: delivery.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}}
|
node := engcomp.Composed{Node: "n1", Own: engcomp.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}}
|
||||||
d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -51,6 +63,7 @@ func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
|||||||
|
|
||||||
// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration.
|
// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration.
|
||||||
func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
||||||
|
t.Run("a fill while the agent restarts", aFillWhileTheAgentRestartsIsWrittenAfterwards)
|
||||||
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil
|
return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil
|
||||||
})
|
})
|
||||||
@@ -59,15 +72,105 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
|||||||
if err != nil || d.Provisioner != "from-agent" {
|
if err != nil || d.Provisioner != "from-agent" {
|
||||||
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// And the health loop's judges are handed what runs once a module settles (g14 and shanks, 2026-10-30 16:14):
|
||||||
|
// restic's composition dropped its process restic.measure, which stayed watched and said down on every look.
|
||||||
|
j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), fakeRuntime{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
judging = j
|
||||||
|
defer func() { judging = nil }()
|
||||||
|
process := func(name string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": "restic." + name, "type": "process", "name": "restic-" + name,
|
||||||
|
"source": "https://store.invalid/restic", "digest": "sha256:" + strings.Repeat("a", 64),
|
||||||
|
"run": []string{"./restic"}, "module": "restic"})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
runs []string
|
||||||
|
watch string
|
||||||
|
}{{[]string{"night", "measure"}, "restic.measure restic.night"}, {[]string{"night"}, "restic.night"}} {
|
||||||
|
var raws []json.RawMessage
|
||||||
|
for _, name := range row.runs {
|
||||||
|
raws = append(raws, process(name))
|
||||||
|
}
|
||||||
|
runs, err := whatRuns([]engcomp.Composed{{Node: "n1", Module: "restic", Resources: raws}}, false,
|
||||||
|
func() (*declaration.Declaration, error) { return nil, errors.New("none kept") })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { return runs, nil })
|
||||||
|
judgeWhatRunsNow(filepath.Join(t.TempDir(), "state.json"), nil)
|
||||||
|
st, _ := j.Look(context.Background())
|
||||||
|
var watched []string
|
||||||
|
for _, r := range st.Resources {
|
||||||
|
watched = append(watched, r.ID)
|
||||||
|
}
|
||||||
|
sort.Strings(watched)
|
||||||
|
if got := strings.Join(watched, " "); got != row.watch {
|
||||||
|
t.Errorf("running %v, the judge watches %q, want %q", row.runs, got, row.watch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
|
||||||
|
// file is there and readable and the controller's key is dropped; until then it reads it.
|
||||||
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
|
||||||
|
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
|
||||||
|
t.Fatalf("before the read is retired: %+v, %v", d, err)
|
||||||
|
}
|
||||||
|
placed := meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
|
||||||
|
// Issue 774, 2026-10-30 19:43: a have naming only a build of the old form (systemd-resolved on shanks, folded in the
|
||||||
|
// node's own) keeps the kept declaration read, since it is the one record of what runs of it.
|
||||||
|
newForm := nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning}
|
||||||
|
oldForm := nodeengine.HaveByNodeModuleValue{Build: "build-1791404191702719760", Phase: nodeengineclient.PhaseRunning}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
has map[string]nodeengine.HaveByNodeModuleValue
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"every placed module has a have of the new form", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm, "lemurs": newForm}, true},
|
||||||
|
{"one has none yet", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm}, false},
|
||||||
|
{"one runs at a build of the old form", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm, "lemurs": oldForm}, false},
|
||||||
|
} {
|
||||||
|
if got, why := allHave(placed, func(m string) (nodeengine.HaveByNodeModuleValue, bool, error) { h, ok := row.has[m]; return h, ok, nil }); got != row.want {
|
||||||
|
t.Errorf("%s: %v (%s)", row.name, got, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The module the mesh provides never has a fingerprint, and keeps no read (review of issue 774).
|
||||||
|
withWireguard := meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "zsh"}, {Module: overlay.Name}}}
|
||||||
|
if got, why := allHave(withWireguard, func(m string) (nodeengine.HaveByNodeModuleValue, bool, error) {
|
||||||
|
if m == overlay.Name {
|
||||||
|
return oldForm, true, nil
|
||||||
|
}
|
||||||
|
return newForm, true, nil
|
||||||
|
}); !got {
|
||||||
|
t.Errorf("%s without a fingerprint beside zsh at the new form keeps the read: %s", overlay.Name, why)
|
||||||
|
}
|
||||||
|
if got, _ := allHave(meshmoduleassigner.Assignments{Node: "n1"}, nil); got {
|
||||||
|
t.Error("a node with nothing placed retires the read")
|
||||||
|
}
|
||||||
|
keptRetired.Store(true)
|
||||||
|
defer keptRetired.Store(false)
|
||||||
|
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
|
||||||
|
t.Errorf("after the read is retired: %v", err)
|
||||||
|
}
|
||||||
|
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
|
||||||
|
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
||||||
// the module's: the declaration joined of both names each file once, so the filling writes it; the node's own resources
|
// the module's: the declaration joined of both names each file once, so the filling writes it; the node's own resources
|
||||||
// no module declares stay. (Live from 2026-10-25 12:43: "2 resources declare X" on every node.)
|
// no module declares stay. (Live from 2026-10-25 12:43: "2 resources declare X" on every node.)
|
||||||
func TestTheNodesOwnCopyOfAModulesResourceIsLeftOut(t *testing.T) {
|
func TestTheNodesOwnCopyOfAModulesResourceIsLeftOut(t *testing.T) {
|
||||||
own := delivery.Composed{Node: "n1", Resources: []json.RawMessage{aFileOf("zsh", "rc"), aFileOf("mesh-wireguard", "conf")},
|
own := engcomp.Composed{Node: "n1", Resources: []json.RawMessage{aFileOf("zsh", "rc"), aFileOf("mesh-wireguard", "conf")},
|
||||||
Filled: []string{"/tmp/r6/zsh/rc"}}
|
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||||
zsh := delivery.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")},
|
zsh := engcomp.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")},
|
||||||
Filled: []string{"/tmp/r6/zsh/rc"}}
|
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||||
d, err := declarationOf(own, zsh)
|
d, err := declarationOf(own, zsh)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -85,7 +188,7 @@ func TestTheNodesOwnCopyOfAModulesResourceIsLeftOut(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The same at the node's own apply: what the others declare is theirs.
|
// The same at the node's own apply: what the others declare is theirs.
|
||||||
trimmed, dropped := ownWithout(own, []delivery.Composed{zsh})
|
trimmed, dropped := ownWithout(own, []engcomp.Composed{zsh})
|
||||||
if len(trimmed.Resources) != 1 || len(dropped) != 1 || dropped[0] != "zsh.rc (zsh)" {
|
if len(trimmed.Resources) != 1 || len(dropped) != 1 || dropped[0] != "zsh.rc (zsh)" {
|
||||||
t.Errorf("the node's own without zsh's: %d resources, dropped %v", len(trimmed.Resources), dropped)
|
t.Errorf("the node's own without zsh's: %d resources, dropped %v", len(trimmed.Resources), dropped)
|
||||||
}
|
}
|
||||||
@@ -115,9 +218,9 @@ func TestTheKeptDeclarationsFilledPathsGoToTheirModules(t *testing.T) {
|
|||||||
strings.Join(filled[agent.NodeModule], ",") != "/etc/motd" || len(filled["zsh"]) != 0 {
|
strings.Join(filled[agent.NodeModule], ",") != "/etc/motd" || len(filled["zsh"]) != 0 {
|
||||||
t.Fatalf("filled by module: %v", filled)
|
t.Fatalf("filled by module: %v", filled)
|
||||||
}
|
}
|
||||||
cc := delivery.Composed{Node: "n1", Module: "claude-code", Resources: by["claude-code"], Filled: filled["claude-code"]}
|
cc := engcomp.Composed{Node: "n1", Module: "claude-code", Resources: by["claude-code"], Filled: filled["claude-code"]}
|
||||||
node := delivery.Composed{Node: "n1"}
|
node := engcomp.Composed{Node: "n1"}
|
||||||
d, err := whatRuns([]delivery.Composed{cc, node}, true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
d, err := whatRuns([]engcomp.Composed{cc, node}, true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||||
if err != nil || !d.Fills("/etc/cc.json") {
|
if err != nil || !d.Fills("/etc/cc.json") {
|
||||||
t.Fatalf("what runs does not name claude-code's file filled: %v (%v)", d, err)
|
t.Fatalf("what runs does not name claude-code's file filled: %v (%v)", d, err)
|
||||||
}
|
}
|
||||||
@@ -130,10 +233,73 @@ func TestTheKeptDeclarationsFilledPathsGoToTheirModules(t *testing.T) {
|
|||||||
// unit or a container of the same name) is not the module's, and is neither dropped nor left to be removed.
|
// unit or a container of the same name) is not the module's, and is neither dropped nor left to be removed.
|
||||||
func TestASharedTargetWithAnotherIDIsNotDropped(t *testing.T) {
|
func TestASharedTargetWithAnotherIDIsNotDropped(t *testing.T) {
|
||||||
ownFile, _ := json.Marshal(map[string]any{"id": "host.zshrc", "type": "file", "path": "/tmp/r6/zsh/rc", "content": "x"})
|
ownFile, _ := json.Marshal(map[string]any{"id": "host.zshrc", "type": "file", "path": "/tmp/r6/zsh/rc", "content": "x"})
|
||||||
own := delivery.Composed{Node: "n1", Resources: []json.RawMessage{ownFile}}
|
own := engcomp.Composed{Node: "n1", Resources: []json.RawMessage{ownFile}}
|
||||||
zsh := delivery.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")}}
|
zsh := engcomp.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")}}
|
||||||
trimmed, dropped := ownWithout(own, []delivery.Composed{zsh})
|
trimmed, dropped := ownWithout(own, []engcomp.Composed{zsh})
|
||||||
if len(dropped) != 0 || len(trimmed.Resources) != 1 {
|
if len(dropped) != 0 || len(trimmed.Resources) != 1 {
|
||||||
t.Errorf("a resource sharing only its target was dropped: %v", dropped)
|
t.Errorf("a resource sharing only its target was dropped: %v", dropped)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 924: the bus link drops, the agent restarts, and the filling is asked while it holds nothing yet. What runs is
|
||||||
|
// then the last view the node held, never the retired kept declaration; and a fill refused for want of any is owed and
|
||||||
|
// told once the agent holds the node again, so every file is written afterwards.
|
||||||
|
func aFillWhileTheAgentRestartsIsWrittenAfterwards(t *testing.T) {
|
||||||
|
keptRetired.Store(true)
|
||||||
|
defer keptRetired.Store(false)
|
||||||
|
node := engcomp.Composed{Node: "n1", Own: engcomp.NodeOwn{Provisioner: "mesh-node-provisioner"}}
|
||||||
|
files := []engcomp.Composed{{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}, Filled: []string{"/tmp/r6/db/a"}}}
|
||||||
|
view := &runsView{everything: append(files, node), holdsNode: true}
|
||||||
|
|
||||||
|
var placed []string
|
||||||
|
// The agent holds the node, then the link drops and it stops: set(nil) keeps its view.
|
||||||
|
holding := true
|
||||||
|
current := &agentNow{view: func(*agent.Agent) *runsView {
|
||||||
|
if holding {
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
return &runsView{}
|
||||||
|
}}
|
||||||
|
current.set(agent.New(agent.Env{}))
|
||||||
|
e := &filling.Engine{
|
||||||
|
Declared: func() (*declaration.Declaration, error) { return runsFromAgent(current)("/nowhere", nil) },
|
||||||
|
UIDOf: func(string) (uint32, error) { return 4242, nil },
|
||||||
|
Lock: func() (func(), error) { return func() {}, nil },
|
||||||
|
Place: func(_ context.Context, _ *declaration.Declaration, path string, _ []byte) (string, string, error) {
|
||||||
|
placed = append(placed, path)
|
||||||
|
return "updated", "d", nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fill := func() filling.Answer {
|
||||||
|
return e.Write(context.Background(), 4242, filling.Request{Path: "/tmp/r6/db/a", Content: []byte("x")})
|
||||||
|
}
|
||||||
|
socket := filepath.Join(t.TempDir(), "sock")
|
||||||
|
if err := os.WriteFile(socket, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The next agent has read no haves and the previous one's view is gone (a cold start): refused, and owed.
|
||||||
|
cold := &agentNow{}
|
||||||
|
cold.set(agent.New(agent.Env{}))
|
||||||
|
cold.set(nil)
|
||||||
|
e.Declared = func() (*declaration.Declaration, error) { return runsFromAgent(cold)("/nowhere", nil) }
|
||||||
|
if a := fill(); a.Failed == "" && a.Refused == "" {
|
||||||
|
t.Fatalf("a fill with nothing read was accepted: %+v", a)
|
||||||
|
}
|
||||||
|
// The link drops: set(nil) keeps the view the agent held, and the fill during the restart is written.
|
||||||
|
e.Declared = func() (*declaration.Declaration, error) { return runsFromAgent(current)("/nowhere", nil) }
|
||||||
|
current.set(nil)
|
||||||
|
if told, err := e.Knock(socket); err != nil || told == "" {
|
||||||
|
t.Fatalf("the refused fill was not owed and told: %q, %v", told, err)
|
||||||
|
}
|
||||||
|
if a := fill(); a.Failed != "" || a.Refused != "" || len(placed) != 1 {
|
||||||
|
t.Fatalf("the fill during the restart was not written: %+v, placed %v", a, placed)
|
||||||
|
}
|
||||||
|
// The new agent holds the node: its own view answers and the kept one is dropped.
|
||||||
|
next := agent.New(agent.Env{})
|
||||||
|
current.set(next)
|
||||||
|
holding = true
|
||||||
|
if _, err := runsFromAgent(current)("/nowhere", nil); err != nil || current.lastView() != nil {
|
||||||
|
t.Fatalf("the kept view stays after the new agent holds the node: %v, %v", err, current.lastView())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
|
|||||||
|
|
||||||
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
||||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
||||||
host, _ := os.Hostname()
|
|
||||||
w := &witness.Watcher{
|
w := &witness.Watcher{
|
||||||
Root: apply.DaemonRoot(), Node: node, Host: host,
|
Root: apply.DaemonRoot(), Node: node,
|
||||||
Asker: func() link.Asker { return queue.Asker() },
|
Asker: func() link.Asker { return queue.Asker() },
|
||||||
Run: witness.Runner(apply.ExecRunner),
|
Run: witness.Runner(apply.ExecRunner),
|
||||||
Hold: func(f func()) {
|
Hold: func(f func()) {
|
||||||
|
|||||||
@@ -3,7 +3,26 @@ module github.com/novox/mesh-host
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936
|
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
@@ -11,12 +30,25 @@ require (
|
|||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||||
github.com/google/go-tpm v0.9.8 // indirect
|
github.com/google/go-tpm v0.9.8 // indirect
|
||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.1 // indirect
|
||||||
github.com/minio/highwayhash v1.0.4 // indirect
|
github.com/minio/highwayhash v1.0.4 // indirect
|
||||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/time v0.15.0 // indirect
|
golang.org/x/time v0.16.0 // indirect
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
|
||||||
|
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,21 +1,13 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSKjgKbWOCOgOJY0WlaDh1n8RcGhvGY/dwXsBQ=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 h1:HGBoWimZXYZp+Qn9VCstYDitOP84t9EUQYd1yM9j2eg=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
@@ -29,5 +21,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
|||||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||||
|
|||||||
@@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() {
|
|||||||
c.forgetProgress()
|
c.forgetProgress()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does).
|
||||||
|
// Nil c runs none.
|
||||||
|
func (c *SetuidCache) Running() bool {
|
||||||
|
if c == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.running
|
||||||
|
}
|
||||||
|
|
||||||
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
||||||
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
||||||
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
||||||
|
|||||||
+1338
-131
File diff suppressed because it is too large
Load Diff
+1311
-167
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,328 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/judge"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-engine seat's read verb composition (issue 774): one module's composition beside what runs of it, so why a
|
||||||
|
// record module waits for a yes, or what its next apply would change, is read from the engine and not guessed from its
|
||||||
|
// have's words. It changes nothing.
|
||||||
|
|
||||||
|
// CompositionView is what the verb composition answers.
|
||||||
|
type CompositionView struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Want is the module's want as the agent holds it; nil when it holds none.
|
||||||
|
Want *WantView `json:"want,omitempty"`
|
||||||
|
// Attempt is what the agent holds of the module's last apply that did not end running: failed, failed for a moment
|
||||||
|
// of the bus, or held for a failed backup, with the want's revision that hold took as answered; nil when none
|
||||||
|
// (issue 1585: why a module stays held was not readable from the engine).
|
||||||
|
Attempt *AttemptView `json:"attempt,omitempty"`
|
||||||
|
// Started says the agent has looked at the module once since it started (its baseline): what runs was taken from its
|
||||||
|
// have, or it was applied.
|
||||||
|
Started bool `json:"started"`
|
||||||
|
// Have is what the module's have key says now; nil when it says nothing.
|
||||||
|
Have *HaveView `json:"have,omitempty"`
|
||||||
|
// Composition is the revision read: the latest of the want's build, or the one whose digest was asked.
|
||||||
|
Composition RevisionView `json:"composition"`
|
||||||
|
// Revisions are the revisions of its key the bucket keeps, oldest first, each with its digest.
|
||||||
|
Revisions []RevisionView `json:"revisions,omitempty"`
|
||||||
|
// Runs is what runs of the module, and where that is known from; nil when nothing of it runs.
|
||||||
|
Runs *RunsView `json:"runs,omitempty"`
|
||||||
|
// RunsManifest is the manifest of the build that runs of the module, as the one place reads it (design 47 G A-484): which
|
||||||
|
// source answered, or the words naming the three it tried. Said also where the want's build has no composition.
|
||||||
|
RunsManifest *RunsManifestView `json:"runs-manifest,omitempty"`
|
||||||
|
// Judged is the move from what runs to the composition read, as the agent judges it before an apply.
|
||||||
|
Judged JudgedView `json:"judged"`
|
||||||
|
// Diff is every resource that differs from what runs, when asked.
|
||||||
|
Diff []judge.ResourceChange `json:"diff,omitempty"`
|
||||||
|
// Own is, of the node's own composition (module node), its own fields beside what runs names (issue 774: ace's
|
||||||
|
// own composition dropped the provisioner client's account, and nothing read could show it).
|
||||||
|
Own *OwnView `json:"own,omitempty"`
|
||||||
|
// Withheld are the module's marked subjects the grants part withholds from this node's runtime, each with why, read
|
||||||
|
// through the seat mesh-grants' verb withheld; WithheldUnknown says when that verb did not answer (design 46 approval
|
||||||
|
// mark §6, lead answer A-239).
|
||||||
|
Withheld []WithheldView `json:"withheld,omitempty"`
|
||||||
|
WithheldUnknown string `json:"withheld-unknown,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithheldView is one marked subject withheld, and why.
|
||||||
|
type WithheldView struct {
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OwnView is the node's own fields of a composition (engcomp.NodeOwn) beside what runs of them, and what the node's
|
||||||
|
// composer held in its last pass.
|
||||||
|
type OwnView struct {
|
||||||
|
// Provisioner is the account the provisioner's client runs as, as the revision read names it; RunsProvisioner as
|
||||||
|
// what runs names it. Empty: the declaration names none, and every filled write is refused.
|
||||||
|
Provisioner string `json:"provisioner_account"`
|
||||||
|
RunsProvisioner string `json:"runs_provisioner_account,omitempty"`
|
||||||
|
// Adoption is the revision's adoption: the node adopted, and which of its modules are taken; nil for a converged node.
|
||||||
|
Adoption *declaration.Adoption `json:"adoption,omitempty"`
|
||||||
|
// Identities, Groups and Libraries count what the revision names of each.
|
||||||
|
Identities int `json:"identities,omitempty"`
|
||||||
|
Groups int `json:"groups,omitempty"`
|
||||||
|
Libraries int `json:"libraries,omitempty"`
|
||||||
|
// Held is each module the node's composer held in its last pass, with why.
|
||||||
|
Held map[string]string `json:"held,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WantView is a want as the verb shows it: as the agent holds it, at the revision of its key it last read.
|
||||||
|
type WantView struct {
|
||||||
|
FP string `json:"fp"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Yes string `json:"yes,omitempty"`
|
||||||
|
Removed bool `json:"removed,omitempty"`
|
||||||
|
// Self says the node composes the module itself (ADR 0547 §2).
|
||||||
|
Self bool `json:"self,omitempty"`
|
||||||
|
// Backup is the operator's answer to a failed backup the want carries, again or skip (ADR 0539 §16); empty when none.
|
||||||
|
Backup string `json:"backup,omitempty"`
|
||||||
|
Words string `json:"words,omitempty"`
|
||||||
|
// Revision is the revision of the want's key the agent read last.
|
||||||
|
Revision uint64 `json:"revision"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AttemptView is the agent's record of a module's last apply that did not end running.
|
||||||
|
type AttemptView struct {
|
||||||
|
FP string `json:"fp"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
// WantRevision and CompositionRevision are the revisions of the want's and the composition's keys it was tried at.
|
||||||
|
WantRevision uint64 `json:"want_revision"`
|
||||||
|
CompositionRevision uint64 `json:"composition_revision"`
|
||||||
|
// Held says it is held for a failed backup; AnsweredRevision is the want's revision the hold took, so an answer
|
||||||
|
// (again or skip) in a later revision releases it.
|
||||||
|
Held bool `json:"held,omitempty"`
|
||||||
|
AnsweredRevision uint64 `json:"answered_revision,omitempty"`
|
||||||
|
// Transient says it failed because the bus did not answer for a moment, and is tried again by the clock.
|
||||||
|
Transient bool `json:"transient,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HaveView is a have as the verb shows it.
|
||||||
|
type HaveView struct {
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Phase string `json:"phase"`
|
||||||
|
Detail string `json:"detail,omitempty"`
|
||||||
|
Judgement string `json:"judgement,omitempty"`
|
||||||
|
Waits string `json:"waits,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RevisionView is one revision of a composition key.
|
||||||
|
type RevisionView struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Revision uint64 `json:"revision,omitempty"`
|
||||||
|
Digest string `json:"digest,omitempty"`
|
||||||
|
Resources int `json:"resources,omitempty"`
|
||||||
|
Latest bool `json:"latest,omitempty"`
|
||||||
|
// Refused is why the revision cannot be read or trusted.
|
||||||
|
Refused string `json:"refused,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunsView is what runs of a module.
|
||||||
|
type RunsView struct {
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Phase string `json:"phase,omitempty"`
|
||||||
|
// From is applied (what this engine applied), have (what its have named at the start) or kept (the declaration the
|
||||||
|
// node kept before the switch).
|
||||||
|
From string `json:"from"`
|
||||||
|
Resources int `json:"resources"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunsManifestView is what the one place answers of the build a module runs here.
|
||||||
|
type RunsManifestView struct {
|
||||||
|
// Source is the source that answered; empty when none did.
|
||||||
|
Source string `json:"source,omitempty"`
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
|
Words string `json:"words"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JudgedView is a move's judgement.
|
||||||
|
type JudgedView struct {
|
||||||
|
Judgement string `json:"judgement"`
|
||||||
|
Words string `json:"words"`
|
||||||
|
// WaitsForYes says the agent would wait for a person's yes before applying it: a module of policy record, a move that
|
||||||
|
// recreates a container, moves data or cannot be judged, and no yes in its want for this digest.
|
||||||
|
WaitsForYes bool `json:"waits_for_yes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composition reads one module's composition beside what runs of it: the want's build (else the have's), its latest
|
||||||
|
// revision or the one whose digest is given, and the move's judgement; with diff, the resources that differ.
|
||||||
|
func (a *Agent) Composition(ctx context.Context, module, digest string, diff bool) (CompositionView, error) {
|
||||||
|
if module == "" {
|
||||||
|
return CompositionView{}, errors.New("name the module, or node for the node's own composition")
|
||||||
|
}
|
||||||
|
view := CompositionView{Node: a.env.Node, Module: module}
|
||||||
|
var have *Have
|
||||||
|
if a.env.Haves != nil {
|
||||||
|
if h, ok := a.env.Haves.Read(ctx, module); ok {
|
||||||
|
have = &h
|
||||||
|
phase := h.Phase
|
||||||
|
if phase == "" {
|
||||||
|
phase = string(nodeengineclient.PhaseRunning)
|
||||||
|
}
|
||||||
|
view.Have = &HaveView{FP: h.FP, Composed: h.Composed, Phase: phase, Detail: h.Detail, Judgement: h.Judgement, Waits: h.Waits}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
w := a.wants[module]
|
||||||
|
fp := ""
|
||||||
|
if w.Present {
|
||||||
|
fp = w.Want.Fp
|
||||||
|
view.Want = &WantView{FP: w.Want.Fp, Composed: w.Want.Composed, Yes: w.Want.Yes, Removed: w.Want.Removed,
|
||||||
|
Self: w.Want.Self, Backup: w.Want.Backup, Words: w.Want.Words, Revision: w.Rev}
|
||||||
|
} else if have != nil {
|
||||||
|
fp = have.FP
|
||||||
|
}
|
||||||
|
if att, ok := a.attempt[module]; ok {
|
||||||
|
view.Attempt = &AttemptView{FP: att.FP, Composed: att.Composed, WantRevision: att.WantRev, CompositionRevision: att.CompRev,
|
||||||
|
Held: att.Held, AnsweredRevision: att.AnsweredRev, Transient: att.Transient}
|
||||||
|
}
|
||||||
|
view.Started = a.checked[module]
|
||||||
|
latest, haveLatest := a.compositionFor(module, fp)
|
||||||
|
r, ran := a.running[module]
|
||||||
|
from, known := a.runningComposition(module)
|
||||||
|
source := "applied"
|
||||||
|
if _, ok := a.applied[module]; !ok {
|
||||||
|
if _, ok := a.had[module]; ok {
|
||||||
|
source = "have"
|
||||||
|
} else {
|
||||||
|
source = "kept"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
// Read outside the lock: the one place reads what the agent runs (Agent.Running takes it), and nothing it does is asked of the bus.
|
||||||
|
if a.env.RunningManifest != nil && module != NodeModule {
|
||||||
|
src, rfp, words := a.env.RunningManifest(module)
|
||||||
|
view.RunsManifest = &RunsManifestView{Source: src, FP: rfp, Words: words}
|
||||||
|
}
|
||||||
|
if module != NodeModule && fp == "" {
|
||||||
|
return view, fmt.Errorf("%s has no want and no have on %s, so no build of it is composed here", module, a.env.Node)
|
||||||
|
}
|
||||||
|
key, err := a.compositionKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return view, err
|
||||||
|
}
|
||||||
|
revs, err := a.revisionsOf(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return view, fmt.Errorf("the revisions of %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
var read *compNow
|
||||||
|
for i := range revs {
|
||||||
|
c := revs[i]
|
||||||
|
rv := RevisionView{Key: key, Revision: c.Rev, Digest: c.Dig, Resources: len(c.C.Resources),
|
||||||
|
Latest: haveLatest && c.Rev == latest.Rev}
|
||||||
|
if c.Err != nil {
|
||||||
|
rv.Refused = c.Err.Error()
|
||||||
|
}
|
||||||
|
view.Revisions = append(view.Revisions, rv)
|
||||||
|
if digest != "" && c.Err == nil && c.Dig == digest {
|
||||||
|
read = &revs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case digest != "" && read == nil && haveLatest && latest.Err == nil && latest.Dig == digest:
|
||||||
|
read = &latest
|
||||||
|
case digest != "" && read == nil:
|
||||||
|
return view, fmt.Errorf("no revision of %s the bucket keeps has the digest %s; revisions lists those it keeps", key, digest)
|
||||||
|
case read == nil && !haveLatest:
|
||||||
|
// What runs of the module is still said, through the one place (A-484): the composition read is the want's build, and a
|
||||||
|
// module held at another build has none of it.
|
||||||
|
words := fmt.Sprintf("the bucket holds no composition of %s at %s", module, fp)
|
||||||
|
if a.env.SelfComposed && a.env.WhyUnmade != nil && module != NodeModule {
|
||||||
|
if why := a.env.WhyUnmade(module); why != "" {
|
||||||
|
words = fmt.Sprintf("the bucket holds no composition of %s at %s; the agent waits for the composer: %s", module, fp, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if view.RunsManifest == nil {
|
||||||
|
return view, errors.New(words)
|
||||||
|
}
|
||||||
|
view.Composition = RevisionView{Key: key, Refused: words}
|
||||||
|
return view, nil
|
||||||
|
case read == nil:
|
||||||
|
read = &latest
|
||||||
|
}
|
||||||
|
view.Composition = RevisionView{Key: key, Revision: read.Rev, Digest: read.Dig, Resources: len(read.C.Resources),
|
||||||
|
Latest: haveLatest && read.Rev == latest.Rev}
|
||||||
|
if read.Err != nil {
|
||||||
|
view.Composition.Refused = read.Err.Error()
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
var running *engcomp.Composed
|
||||||
|
if known {
|
||||||
|
running = &from
|
||||||
|
rv := &RunsView{From: source, Resources: len(from.Resources), FP: from.FP}
|
||||||
|
if ran {
|
||||||
|
rv.FP, rv.Composed, rv.Phase = r.Fp, r.Composed, string(r.Phase)
|
||||||
|
}
|
||||||
|
view.Runs = rv
|
||||||
|
}
|
||||||
|
if module == NodeModule {
|
||||||
|
o := read.C.Own
|
||||||
|
view.Own = &OwnView{Provisioner: o.Provisioner, Adoption: o.Adoption, Identities: len(o.Identities), Groups: len(o.Groups),
|
||||||
|
Libraries: len(o.Libraries)}
|
||||||
|
if running != nil {
|
||||||
|
view.Own.RunsProvisioner = running.Own.Provisioner
|
||||||
|
}
|
||||||
|
if a.env.Held != nil {
|
||||||
|
view.Own.Held = a.env.Held()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
j := judge.JudgeMove(running, read.C)
|
||||||
|
view.Judged = JudgedView{Judgement: string(j.Judgement), Words: j.Words}
|
||||||
|
if a.env.SelfComposed && a.env.Policy != nil && module != NodeModule && (!w.Present || w.Want.Yes != read.Dig) {
|
||||||
|
switch j.Judgement {
|
||||||
|
case nodeengineclient.JudgedRecreates, nodeengineclient.JudgedMovesData, nodeengineclient.JudgedCannot:
|
||||||
|
view.Judged.WaitsForYes = a.env.Policy(ctx, module) == catalogue.PolicyRecord
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if diff {
|
||||||
|
view.Diff = judge.ChangedResources(running, read.C)
|
||||||
|
}
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositionKey is a module's composition key at a build, or the node's own.
|
||||||
|
func (a *Agent) compositionKey(module, fp string) (string, error) {
|
||||||
|
if module == NodeModule {
|
||||||
|
return engcomp.NodeKey(a.env.Node)
|
||||||
|
}
|
||||||
|
return engcomp.Key(a.env.Node, module, fp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// revisionsOf is every revision of a composition key the state keeps, oldest first, each with its digest (unsigned, ADR
|
||||||
|
// 0593 §6), or why it is refused. Nil Revisions
|
||||||
|
// reads none.
|
||||||
|
func (a *Agent) revisionsOf(ctx context.Context, key string) ([]compNow, error) {
|
||||||
|
if a.env.Revisions == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
revs, err := a.env.Revisions(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []compNow
|
||||||
|
for _, u := range revs {
|
||||||
|
if u.Deleted || u.Key != key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, err := engcomp.Decode(u.Value)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, compNow{Raw: u.Value, Rev: u.Revision, Err: err})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
dig, err := engcomp.DigestOf(c) // unsigned: a record, never verified (ADR 0593 §6)
|
||||||
|
out = append(out, compNow{Raw: u.Value, C: c, Dig: dig, Rev: u.Revision, Err: err})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,372 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G
|
||||||
|
// §296, §297, §554, §555, §560, §630 to §632, the lead's answers A-77, A-98 and A-108): the node agent asks the seat
|
||||||
|
// `mesh-recovery`'s `snapshot-kept` and waits for the `ended` of its own act, or, at a revert, stops the newer build, asks
|
||||||
|
// `kept-points`, asks `restore-kept` with the snapshot of the point of the form it reverts from, waits for its `restored`, and
|
||||||
|
// after the reverted build's apply calls the point's catch-up as the module's tool.
|
||||||
|
|
||||||
|
// Ending is the ending event of an act of the recovery module as the node agent reads it: `ended` for a restore point taken,
|
||||||
|
// `restored` for one restored (design 47 G §560).
|
||||||
|
type Ending struct {
|
||||||
|
Act string
|
||||||
|
Outcome string
|
||||||
|
Error string
|
||||||
|
Snapshot string
|
||||||
|
// Why is the hand act's words for a cancel (design 47 G A-117, §560).
|
||||||
|
Why string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecoveryWaitMinutes is the default of the manifest's config key `recovery-wait-minutes`, the bound of the wait for an
|
||||||
|
// ending event, counted from the first ask (design 47 G §554, §633).
|
||||||
|
const RecoveryWaitMinutes = 90
|
||||||
|
|
||||||
|
// Recovery is what the node agent needs of the recovery module and the manifests to move kept data. Nil moves none.
|
||||||
|
type Recovery struct {
|
||||||
|
// FormChanges are the kept data whose form differs between the running build's manifest and the applied build's (the
|
||||||
|
// catalog half's entries by fingerprint); an error says it is not known.
|
||||||
|
FormChanges func(ctx context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error)
|
||||||
|
// The seat's verbs, asked in the node-engine's own name; an error is the ask not answered.
|
||||||
|
SnapshotKept func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error)
|
||||||
|
RestoreKept func(ctx context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error)
|
||||||
|
KeptPoints func(ctx context.Context, in meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error)
|
||||||
|
// Hear follows the ending events of kind "ended" or "restored" of one module's kept data: the last one said already first,
|
||||||
|
// then each after it, until stop is called. It is made before the verb is asked (design 47 G §560).
|
||||||
|
Hear func(ctx context.Context, kind, module, kept string) (events <-chan Ending, stop func(), err error)
|
||||||
|
// Bound writes the bound key `<what>.<id>` of a wait in the state bounds with its time to live, so its end is the bus's
|
||||||
|
// marker, heard as expired (design 47 G A-77, A-92); end deletes it. Nil keeps no bound.
|
||||||
|
Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error)
|
||||||
|
// CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108).
|
||||||
|
CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error
|
||||||
|
// WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes.
|
||||||
|
WaitMinutes int
|
||||||
|
// Now is the clock; nil reads the machine's.
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Recovery) now() time.Time {
|
||||||
|
if r.Now != nil {
|
||||||
|
return r.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Recovery) bound() time.Duration {
|
||||||
|
if r.WaitMinutes > 0 {
|
||||||
|
return time.Duration(r.WaitMinutes) * time.Minute
|
||||||
|
}
|
||||||
|
return RecoveryWaitMinutes * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptMove is one kept data a want moves across a change of its form, as the seat mesh-recovery's verbs name it.
|
||||||
|
type KeptMove struct {
|
||||||
|
// Name is the kept data's local name, Form the form the restore point holds (the older one), and Fingerprint the
|
||||||
|
// fingerprint the restore point is taken before (the newer build's) or reverted from.
|
||||||
|
Name string
|
||||||
|
Form int
|
||||||
|
Fingerprint string
|
||||||
|
// CatchUp is, when taking the point, the tool the newer manifest declares beside the kept data.
|
||||||
|
CatchUp *meshrecovery.CatchUp
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptMoves are the restore points a move needs: a point to take before the newer build starts (the kept data's form rises,
|
||||||
|
// or the newer manifest declares it no more) and a point to restore before the older build starts (the kept data goes back to
|
||||||
|
// a lower form). A kept data the running build does not declare is neither: nothing was written before it.
|
||||||
|
func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (take, restore []KeptMove) {
|
||||||
|
for _, c := range changes {
|
||||||
|
switch {
|
||||||
|
case c.Running == nil:
|
||||||
|
case c.Applied == nil || c.Applied.Form > c.Running.Form:
|
||||||
|
m := KeptMove{Name: c.Name, Form: c.Running.Form, Fingerprint: appliedFp}
|
||||||
|
if c.Applied != nil && c.Applied.CatchUp != nil {
|
||||||
|
m.CatchUp = &meshrecovery.CatchUp{Tool: c.Applied.CatchUp.Tool, Arguments: c.Applied.CatchUp.Arguments}
|
||||||
|
}
|
||||||
|
take = append(take, m)
|
||||||
|
case c.Applied.Form < c.Running.Form:
|
||||||
|
restore = append(restore, KeptMove{Name: c.Name, Form: c.Applied.Form, Fingerprint: runningFp})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return take, restore
|
||||||
|
}
|
||||||
|
|
||||||
|
// recoveryHeld is how a wait for the recovery module ended when the move cannot go on: the words are the reason of the hold.
|
||||||
|
type recoveryHeld struct {
|
||||||
|
Words string
|
||||||
|
Interrupted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
|
||||||
|
// ending of the act it names and asks again, `failed` with `stopped` asks again within the same bound, a failure with any
|
||||||
|
// other error holds. The wait is written in the have with its first ask's time and the act, so a restart waits for that act
|
||||||
|
// and asks nothing again; past the bound the have says the wait, and the wait goes on (the watchman raises it).
|
||||||
|
func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, ask func() (accepted, busy, errWords string, err error)) (Ending, *recoveryHeld) {
|
||||||
|
r := a.env.Recovery
|
||||||
|
module := h.Module
|
||||||
|
step := nodeengine.WaitStepCompose
|
||||||
|
wait := nodeengine.Wait{Owner: nodeengine.Seat, Step: step, On: "mesh-recovery." + kind + "." + module + "." + k.Name,
|
||||||
|
Reason: "recovery waits", Bound: r.bound().String()}
|
||||||
|
// A restart: the have names the wait with its ask time and act.
|
||||||
|
last, ok := a.env.Haves.Read(ctx, module)
|
||||||
|
if ok && last.Wait != nil && last.Wait.On == wait.On && last.Wait.AskAt != nil {
|
||||||
|
wait.AskAt, wait.Act, wait.Since = last.Wait.AskAt, last.Wait.Act, last.Wait.Since
|
||||||
|
}
|
||||||
|
now := r.now()
|
||||||
|
if wait.AskAt == nil {
|
||||||
|
wait.AskAt, wait.Since = &now, now
|
||||||
|
}
|
||||||
|
events, stop, err := r.Hear(ctx, kind, module, k.Name)
|
||||||
|
if err != nil {
|
||||||
|
return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act cannot be heard: " + err.Error()}
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
var expired <-chan struct{}
|
||||||
|
if r.Bound != nil {
|
||||||
|
bound := r.bound() - now.Sub(*wait.AskAt)
|
||||||
|
if bound < time.Second {
|
||||||
|
bound = time.Second
|
||||||
|
}
|
||||||
|
exp, end, berr := r.Bound(ctx, "recovery", module+"."+kind+"."+k.Name, module, *wait.AskAt, bound)
|
||||||
|
if berr != nil {
|
||||||
|
return Ending{}, &recoveryHeld{Words: "the bound of the wait for the recovery module cannot be written: " + berr.Error()}
|
||||||
|
}
|
||||||
|
defer end()
|
||||||
|
expired = exp
|
||||||
|
}
|
||||||
|
write := func() {
|
||||||
|
w := wait
|
||||||
|
h.Wait = &w
|
||||||
|
a.writeHave(ctx, h)
|
||||||
|
}
|
||||||
|
act, busyOn := wait.Act, ""
|
||||||
|
for {
|
||||||
|
if act == "" && busyOn == "" {
|
||||||
|
write() // the first ask's time is in the have before the ask
|
||||||
|
accepted, busy, words, err := ask()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
|
||||||
|
}
|
||||||
|
return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()}
|
||||||
|
case words != "":
|
||||||
|
return Ending{}, &recoveryHeld{Words: words}
|
||||||
|
case accepted != "":
|
||||||
|
act, wait.Act = accepted, accepted
|
||||||
|
write()
|
||||||
|
case busy != "":
|
||||||
|
busyOn = busy
|
||||||
|
default:
|
||||||
|
return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
|
||||||
|
case <-expired:
|
||||||
|
expired = nil
|
||||||
|
wait.Words = fmt.Sprintf("the recovery module has not ended its act within %s", r.bound())
|
||||||
|
write()
|
||||||
|
case ev, open := <-events:
|
||||||
|
if !open {
|
||||||
|
return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act is no longer heard"}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case busyOn != "" && ev.Act == busyOn:
|
||||||
|
busyOn = ""
|
||||||
|
case act != "" && ev.Act == act:
|
||||||
|
switch {
|
||||||
|
case ev.Outcome == meshrecovery.EndedEventOutcomeDone:
|
||||||
|
return ev, nil
|
||||||
|
case ev.Error == "stopped":
|
||||||
|
act, wait.Act = "", ""
|
||||||
|
case ev.Error == "cancelled":
|
||||||
|
words := "cancelled"
|
||||||
|
if ev.Why != "" {
|
||||||
|
words += ": " + ev.Why
|
||||||
|
}
|
||||||
|
return ev, &recoveryHeld{Words: words}
|
||||||
|
default:
|
||||||
|
words := ev.Error
|
||||||
|
if words == "" {
|
||||||
|
words = "the recovery module's act failed"
|
||||||
|
}
|
||||||
|
return ev, &recoveryHeld{Words: words}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it
|
||||||
|
// answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up
|
||||||
|
// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither, and nor does one that answers a held revert live.
|
||||||
|
func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool, others []compositions.Composed) (held bool, catchUps []meshrecovery.CatchUp) {
|
||||||
|
r := a.env.Recovery
|
||||||
|
if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
fp, dig := w.Want.Fp, w.Want.Composed
|
||||||
|
a.mu.Lock()
|
||||||
|
prev, runs := a.running[module]
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !runs {
|
||||||
|
if h, found, _ := a.readHave(ctx, module); found {
|
||||||
|
prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if prev.Fp == "" || prev.Fp == fp {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
changes, err := r.FormChanges(ctx, module, prev.Fp, fp)
|
||||||
|
if err != nil {
|
||||||
|
// Whether the move changes a form is not known: it waits for the entries and never moves without its restore point (design 47 G
|
||||||
|
// A-113). No attempt is kept, so the composer handing the composition again, or any settle, asks again.
|
||||||
|
words := fmt.Sprintf("whether %s's kept data changes form between %s and %s is not known, so it is not moved: %v", module, prev.Fp, fp, err)
|
||||||
|
a.sayOnce("kept:"+module+":"+fp, module+" waits: "+words)
|
||||||
|
since := r.now()
|
||||||
|
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
|
||||||
|
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: since, Words: words}})
|
||||||
|
// The composition of the want is held: the wait is for an entry, not for a composition, so the composer is not told it
|
||||||
|
// is waited for (that handed the same composition again every pass, and the move failed the same way again); it
|
||||||
|
// hands the composition again when the entry answers (Composer.entryChanged).
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
take, restore := keptMoves(changes, prev.Fp, fp)
|
||||||
|
if len(take)+len(restore) == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
// The operator's answer "live" to a held revert, read from this want and from nothing else: the reverted build starts on the live
|
||||||
|
// data as it is, with no restore, no aside taken and no catch-up, and applies as any want (design 47 G section 710).
|
||||||
|
if len(restore) > 0 && w.Want.Revert == meshwants.WantsByWantNodeModuleValueRevertLive {
|
||||||
|
a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert))
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
hold := func(words string, interrupted bool) {
|
||||||
|
if interrupted || ctx.Err() != nil {
|
||||||
|
words = BackupInterrupted
|
||||||
|
a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
|
||||||
|
} else {
|
||||||
|
a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
|
||||||
|
a.mu.Unlock()
|
||||||
|
since := r.now()
|
||||||
|
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
|
||||||
|
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}})
|
||||||
|
}
|
||||||
|
backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)}
|
||||||
|
for _, m := range take {
|
||||||
|
why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp)
|
||||||
|
_, held := a.askAndWait(ctx, backing, "ended", m, func() (string, string, string, error) {
|
||||||
|
got, err := r.SnapshotKept(ctx, meshrecovery.SnapshotKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
|
||||||
|
Fingerprint: m.Fingerprint, CatchUp: m.CatchUp, Why: why})
|
||||||
|
return got.Accepted, got.Busy, got.Error, err
|
||||||
|
})
|
||||||
|
if held != nil {
|
||||||
|
hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(restore) == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
// A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside.
|
||||||
|
stopped := false
|
||||||
|
if a.env.Stop != nil {
|
||||||
|
if res := a.env.Stop(ctx, module); res.Failed {
|
||||||
|
hold("the newer build could not be stopped before the restore: "+res.Detail, false)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
stopped = true
|
||||||
|
}
|
||||||
|
// A revert that cannot restore starts the newer build again and holds the move, saying why: a module is never left stopped
|
||||||
|
// (design 47 G A-113).
|
||||||
|
holdRevert := func(words string, interrupted bool) {
|
||||||
|
if stopped {
|
||||||
|
words += "; " + a.startAgain(ctx, module, prev, others)
|
||||||
|
}
|
||||||
|
hold(words, interrupted)
|
||||||
|
}
|
||||||
|
for _, m := range restore {
|
||||||
|
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name})
|
||||||
|
if err != nil {
|
||||||
|
holdRevert(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
var point *meshrecovery.Point
|
||||||
|
for i := range pts.Points {
|
||||||
|
p := pts.Points[i]
|
||||||
|
if p.Module == module && p.Kept == m.Name && int(p.Form) == m.Form && p.Fingerprint == m.Fingerprint {
|
||||||
|
if point == nil || p.At.After(point.At) {
|
||||||
|
point = &pts.Points[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if point == nil {
|
||||||
|
holdRevert(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint)
|
||||||
|
_, held := a.askAndWait(ctx, backing, "restored", m, func() (string, string, string, error) {
|
||||||
|
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
|
||||||
|
Fingerprint: m.Fingerprint, Snapshot: point.Snapshot, Why: why})
|
||||||
|
return got.Accepted, got.Busy, got.Error, err
|
||||||
|
})
|
||||||
|
if held != nil {
|
||||||
|
holdRevert(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if point.CatchUp != nil {
|
||||||
|
catchUps = append(catchUps, *point.CatchUp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, catchUps
|
||||||
|
}
|
||||||
|
|
||||||
|
// startAgain starts the build a module ran before a revert stopped it, from the composition the agent applied or holds of its
|
||||||
|
// fingerprint, and says whether it did (design 47 G A-113).
|
||||||
|
func (a *Agent) startAgain(ctx context.Context, module string, prev nodeengine.HaveByNodeModuleValue, others []compositions.Composed) string {
|
||||||
|
a.mu.Lock()
|
||||||
|
c, ok := a.applied[module]
|
||||||
|
if !ok || c.C.Module == "" {
|
||||||
|
c, ok = a.compositionFor(module, prev.Fp)
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !ok || a.env.Apply == nil {
|
||||||
|
return "the newer build could not be started again: its composition is not held"
|
||||||
|
}
|
||||||
|
res := a.env.Apply(context.WithoutCancel(ctx), c.C, others)
|
||||||
|
if res.Failed {
|
||||||
|
return "the newer build could not be started again: " + res.Detail
|
||||||
|
}
|
||||||
|
return "the newer build runs again"
|
||||||
|
}
|
||||||
|
|
||||||
|
// catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own
|
||||||
|
// tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand.
|
||||||
|
func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) {
|
||||||
|
if a.env.Recovery == nil || a.env.Recovery.CatchUp == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, c := range calls {
|
||||||
|
if err := a.env.Recovery.CatchUp(ctx, module, c); err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("%s runs the reverted build, and its catch-up tool %s failed: %v", module, c.Tool, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.env.Say(fmt.Sprintf("%s's catch-up tool %s ran after the restore", module, c.Tool))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,326 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets/bucketstest"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A recovery module that answers as scripted: each ask takes the next answer and says the events it carries after it.
|
||||||
|
type recoveryFake struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
asks []string // verb and its input, in order
|
||||||
|
answers []fakeAnswer
|
||||||
|
points []meshrecovery.Point
|
||||||
|
events chan Ending
|
||||||
|
changes []catalogue.FormChange
|
||||||
|
calls []string // stop, ask, apply, catch-up, in order
|
||||||
|
catchUp []meshrecovery.CatchUp
|
||||||
|
expired chan struct{}
|
||||||
|
unknown error // FormChanges answers it
|
||||||
|
waited int // times the agent told the composer it waits for a composition
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeAnswer struct {
|
||||||
|
accepted, busy, errWords string
|
||||||
|
then []Ending
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recoveryFake) next(verb string, in any) (string, string, string) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.asks = append(f.asks, fmt.Sprintf("%s %s", verb, asJSON(in)))
|
||||||
|
f.calls = append(f.calls, verb)
|
||||||
|
if len(f.answers) == 0 {
|
||||||
|
return "", "", "no answer scripted"
|
||||||
|
}
|
||||||
|
a := f.answers[0]
|
||||||
|
f.answers = f.answers[1:]
|
||||||
|
for _, e := range a.then {
|
||||||
|
f.events <- e
|
||||||
|
}
|
||||||
|
return a.accepted, a.busy, a.errWords
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recoveryFake) env() *Recovery {
|
||||||
|
return &Recovery{
|
||||||
|
FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) {
|
||||||
|
return f.changes, f.unknown
|
||||||
|
},
|
||||||
|
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
|
||||||
|
a, b, e := f.next("snapshot-kept", in)
|
||||||
|
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
|
||||||
|
},
|
||||||
|
RestoreKept: func(_ context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) {
|
||||||
|
a, b, e := f.next("restore-kept", in)
|
||||||
|
return meshrecovery.RestoreKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
|
||||||
|
},
|
||||||
|
KeptPoints: func(context.Context, meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.calls = append(f.calls, "kept-points")
|
||||||
|
return meshrecovery.KeptPointsAnswer{Points: f.points}, nil
|
||||||
|
},
|
||||||
|
Hear: func(context.Context, string, string, string) (<-chan Ending, func(), error) {
|
||||||
|
return f.events, func() {}, nil
|
||||||
|
},
|
||||||
|
Bound: func(context.Context, string, string, string, time.Time, time.Duration) (<-chan struct{}, func(), error) {
|
||||||
|
return f.expired, func() {}, nil
|
||||||
|
},
|
||||||
|
CatchUp: func(_ context.Context, _ string, c meshrecovery.CatchUp) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.calls = append(f.calls, "catch-up")
|
||||||
|
f.catchUp = append(f.catchUp, c)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Now: time.Now,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recoveryFake) seen() []string {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return slices.Clone(f.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// moveDB runs a data module at fp "a", then wants it at fp "b", and answers the bed once the second want is settled or held.
|
||||||
|
func moveDB(t *testing.T, f *recoveryFake, ranFrom, to string, settle func(b *bed) bool, before ...func(b *bed, dig string)) *bed {
|
||||||
|
t.Helper()
|
||||||
|
b := newBed(t)
|
||||||
|
inner, applies := b.env.Apply, 0
|
||||||
|
b.env.Apply = func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result {
|
||||||
|
if applies++; applies >= 2 {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.calls = append(f.calls, "apply")
|
||||||
|
f.mu.Unlock()
|
||||||
|
}
|
||||||
|
return inner(ctx, c, others)
|
||||||
|
}
|
||||||
|
f.events, f.expired = make(chan Ending, 16), make(chan struct{})
|
||||||
|
b.env.Recovery = f.env()
|
||||||
|
b.env.Waiting = func(string, string) { f.mu.Lock(); f.waited++; f.mu.Unlock() }
|
||||||
|
b.env.Stop = func(context.Context, string) Result {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.calls = append(f.calls, "stop")
|
||||||
|
f.mu.Unlock()
|
||||||
|
return Result{}
|
||||||
|
}
|
||||||
|
stop := b.run()
|
||||||
|
t.Cleanup(stop)
|
||||||
|
fp1, fp2 := fpOf(ranFrom), fpOf(to)
|
||||||
|
ck, craw, dig := composition(t, "db", fp1, true, file("db", "x", "1"))
|
||||||
|
wk, wraw := wantFor(t, "db", fp1, dig, "")
|
||||||
|
b.comps.put(ck, craw)
|
||||||
|
b.wants.put(wk, wraw)
|
||||||
|
eventually(t, "the first build is not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||||
|
b.idle()
|
||||||
|
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
|
||||||
|
for _, h := range before {
|
||||||
|
h(b, dig)
|
||||||
|
}
|
||||||
|
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
|
||||||
|
b.comps.put(ck2, craw2)
|
||||||
|
b.wants.put(wk2, wraw2)
|
||||||
|
eventually(t, "the move is neither applied nor held", func() bool { return settle(b) })
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// testAMoveAcrossAFormChange is design 47 G §296, §297, §554, §560, §630 to §632 and A-98, A-108 at the node agent.
|
||||||
|
func testAMoveAcrossAFormChange(t *testing.T) {
|
||||||
|
up := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
|
||||||
|
Applied: &catalogue.KeptLayout{Form: 2, CatchUp: &catalogue.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
|
||||||
|
down := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 2},
|
||||||
|
Applied: &catalogue.KeptLayout{Form: 1}}}
|
||||||
|
fp := func(c string) string { return fpOf(c) }
|
||||||
|
done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} }
|
||||||
|
failed := func(act, why string) Ending {
|
||||||
|
e := Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why}
|
||||||
|
if why == "cancelled" {
|
||||||
|
e.Why = "the operator stopped it" // the hand act's words (design 47 G A-117)
|
||||||
|
}
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
applied := func(b *bed) bool { return len(b.appliedList()) == 2 }
|
||||||
|
heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" }
|
||||||
|
|
||||||
|
t.Run("a form that rises is taken as a restore point before the move, with the why and the catch-up, and then applied", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{done("a1")}}}}
|
||||||
|
moveDB(t, f, "a", "b", applied)
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
want := fmt.Sprintf("snapshot-kept %s", asJSON(meshrecovery.SnapshotKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("b"),
|
||||||
|
Why: "form change of db's cache to " + fp("b"), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}))
|
||||||
|
if f.asks[0] != want {
|
||||||
|
t.Fatalf("asked %s, want %s", f.asks[0], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{
|
||||||
|
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
|
||||||
|
{accepted: "a2", then: []Ending{failed("a2", "stopped")}},
|
||||||
|
{accepted: "a3", then: []Ending{done("a3")}},
|
||||||
|
}}
|
||||||
|
moveDB(t, f, "a", "b", applied)
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "snapshot-kept", "snapshot-kept", "apply"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, row := range []struct{ error, held string }{
|
||||||
|
{"cancelled", "cancelled: the operator stopped it"},
|
||||||
|
{"cap", "cap"},
|
||||||
|
{"restore point damaged", "restore point damaged"},
|
||||||
|
{"disk full", "disk full"},
|
||||||
|
} {
|
||||||
|
t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
if got := f.seen(); len(got) != 1 {
|
||||||
|
t.Fatalf("asked again: %v", got)
|
||||||
|
}
|
||||||
|
if n := len(b.appliedList()); n != 1 {
|
||||||
|
t.Fatalf("applied %d", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("the bound passed says the wait in the have and the wait goes on to the ending", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1"}}}
|
||||||
|
b := moveDB(t, f, "a", "b", func(b *bed) bool {
|
||||||
|
b.haves.mu.Lock()
|
||||||
|
defer b.haves.mu.Unlock()
|
||||||
|
for _, h := range b.haves.log {
|
||||||
|
if h.Wait != nil && h.Wait.Words != "" && h.Phase == "backing-up" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, func(*bed, string) { close(f.expired) })
|
||||||
|
f.events <- done("a1")
|
||||||
|
eventually(t, "the move went on after the ending", func() bool { return applied(b) })
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a revert stops the newer build, restores the point of the form it reverts from, applies, then calls its catch-up", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: down,
|
||||||
|
points: []meshrecovery.Point{
|
||||||
|
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("z"), Snapshot: "other", At: time.Now()},
|
||||||
|
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now(), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild"}}},
|
||||||
|
answers: []fakeAnswer{{accepted: "r1", then: []Ending{done("r1")}}}}
|
||||||
|
moveDB(t, f, "a", "b", func(b *bed) bool { return len(f.seen()) >= 5 })
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply", "catch-up"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
if f.catchUp[0].Tool != "rebuild" {
|
||||||
|
t.Fatalf("catch-up %+v", f.catchUp)
|
||||||
|
}
|
||||||
|
want := fmt.Sprintf("restore-kept %s", asJSON(meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"),
|
||||||
|
Snapshot: "snap-1", Why: "revert of db's cache from " + fp("a")}))
|
||||||
|
if f.asks[0] != want {
|
||||||
|
t.Fatalf("asked %s, want %s", f.asks[0], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a revert with no point of its form holds as a restore point missing", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: down}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a held revert, then the want with revert live, starts the older build with no restore, no aside and no catch-up (design 47 G section 710)", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: down}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
before := len(b.appliedList())
|
||||||
|
_, _, dig2 := composition(t, "db", fpOf("b"), true, file("db", "x", "2"))
|
||||||
|
raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpOf("b"), Composed: dig2, Revert: meshwants.WantsByWantNodeModuleValueRevertLive,
|
||||||
|
Words: "the operator answered \"live\" to the revert ask (asking 1)"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.wants.put(meshwants.WantsByWantNodeModuleKey(node, "db"), raw)
|
||||||
|
eventually(t, "the older build runs on the answer", func() bool { h := b.haves.last("db"); return h.FP == fpOf("b") && h.Phase != "held" })
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "apply", "apply"}) && len(b.appliedList()) <= before {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
for _, c := range f.seen() {
|
||||||
|
if c == "restore-kept" || c == "snapshot-kept" || c == "catch-up" {
|
||||||
|
t.Fatalf("calls %v", f.seen())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := strings.Count(strings.Join(f.seen(), " "), "kept-points"); n != 1 {
|
||||||
|
t.Fatalf("the points were read again on the answer: %v", f.seen())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: down,
|
||||||
|
points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}},
|
||||||
|
answers: []fakeAnswer{{accepted: "r1", then: []Ending{failed("r1", "restore point damaged")}}}}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
if h := b.haves.last("db"); !contains(h.Detail, "restore point damaged") || !contains(h.Detail, "the newer build runs again") ||
|
||||||
|
h.Wait == nil || h.Wait.Step != nodeengine.WaitStepCompose {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a form change not known moves nothing and waits for the entries (A-113)", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{unknown: fmt.Errorf("the entry is asked")}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
f.mu.Lock()
|
||||||
|
if f.waited != 0 {
|
||||||
|
t.Errorf("the composer was told the agent waits for a composition %d times; it waits for an entry and holds the composition", f.waited)
|
||||||
|
}
|
||||||
|
f.mu.Unlock()
|
||||||
|
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, "is not known") || h.Wait == nil || h.Wait.Reason != "entry-missing" {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
if n := len(b.appliedList()); n != 1 || len(f.seen()) != 0 {
|
||||||
|
t.Fatalf("applied %d, calls %v", n, f.seen())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up}
|
||||||
|
b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) {
|
||||||
|
at := time.Now().Add(-time.Minute)
|
||||||
|
b.haves.mu.Lock()
|
||||||
|
b.haves.stored["db"] = Have{Module: "db", FP: fp("a"), Phase: "backing-up", Wait: &nodeengine.Wait{
|
||||||
|
On: "mesh-recovery.ended.db.cache", AskAt: &at, Act: "a1", Since: at}}
|
||||||
|
b.haves.mu.Unlock()
|
||||||
|
f.events <- done("a1")
|
||||||
|
})
|
||||||
|
_ = b
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"apply"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(s, sub string) bool { return strings.Contains(s, sub) }
|
||||||
|
|
||||||
|
func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) }
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PartsDiffer is the start of the detail of a have held because a want's parts and the digests its composition fetches
|
||||||
|
// disagree (design 47 G section 524).
|
||||||
|
const PartsDiffer = "parts-differ:"
|
||||||
|
|
||||||
|
// EntryAsked and EntryMissing open the detail of a have held for the entry the want names (design 47 G section 421 and
|
||||||
|
// section 614): the entry is asked of the catalog half and not yet answered, or the half answered that it holds none. Both
|
||||||
|
// are the phase held, and each names the fingerprint.
|
||||||
|
const (
|
||||||
|
EntryAsked = "entry-asked:"
|
||||||
|
EntryMissing = "entry-missing:"
|
||||||
|
)
|
||||||
|
|
||||||
|
// judgedAgain says a held have is a hold the node-engine judges again at a start, not one that waits for the operator's answer:
|
||||||
|
// its wait's reason (the structured field the have carries) is that its want's parts differ from its composition (§524) or its
|
||||||
|
// entry is not held (§421), whatever words its detail opens with. Only a failed backup or restore point waits for an answer (ADR
|
||||||
|
// 0568 §4); novox, 2026-11-16: the indexer held "parts-differ" was kept at a start as "held for a failed backup"; g14, 2026-11-18:
|
||||||
|
// feh held "whether ..." with the reason entry-missing was kept the same way.
|
||||||
|
func judgedAgain(h Have) bool {
|
||||||
|
if h.Wait == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch h.Wait.Reason {
|
||||||
|
case strings.TrimSuffix(PartsDiffer, ":"), strings.TrimSuffix(EntryAsked, ":"), strings.TrimSuffix(EntryMissing, ":"):
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldReason is the opening of a detail held for an entry: EntryAsked or EntryMissing.
|
||||||
|
func heldReason(why string) string {
|
||||||
|
if strings.HasPrefix(why, EntryAsked) {
|
||||||
|
return EntryAsked
|
||||||
|
}
|
||||||
|
return EntryMissing
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldForEntry says whether why is a hold for the entry a want names.
|
||||||
|
func heldForEntry(why string) bool {
|
||||||
|
return strings.HasPrefix(why, EntryAsked) || strings.HasPrefix(why, EntryMissing)
|
||||||
|
}
|
||||||
|
|
||||||
|
// upstreamCopy is whether a routed image is the registry seat's copy of an upstream image, `<address>/upstream/<host>/<image>@<digest>`:
|
||||||
|
// the mesh mirrors it, but it built nothing, so it is no part of a want (design 47 G A-454).
|
||||||
|
func upstreamCopy(image string) bool {
|
||||||
|
_, path, ok := strings.Cut(image, "/")
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, _, upstream := reference.InUpstream(path)
|
||||||
|
return upstream
|
||||||
|
}
|
||||||
|
|
||||||
|
// partsDifference checks a want's `parts` against the digests of the archive, process and container resources of the
|
||||||
|
// composition, which are the digests the apply fetches and checks (design 47 G section 524). A want with no `parts` (an
|
||||||
|
// assigner of before G, or a node whose record does not state `want-parts`) is applied as today: it returns "". With
|
||||||
|
// `parts`, the want is applied only when the digest of every resource fetched is among the values of `parts` and every
|
||||||
|
// value of `parts` is the digest of a resource fetched; otherwise it returns the detail naming each digest.
|
||||||
|
func partsDifference(parts map[string]string, resources []json.RawMessage, templates []string) string {
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
inParts := map[string]bool{}
|
||||||
|
for _, d := range parts {
|
||||||
|
inParts[d] = true
|
||||||
|
}
|
||||||
|
fetched := map[string]bool{}
|
||||||
|
// A templates archive the composition was rendered from was read from the artifact store and checked by its digest:
|
||||||
|
// a part fetched (design 47 G A-481, A-481a).
|
||||||
|
for _, d := range templates {
|
||||||
|
fetched[d] = true
|
||||||
|
}
|
||||||
|
for _, raw := range resources {
|
||||||
|
r, err := declaration.ParseResource(raw)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch res := r.(type) {
|
||||||
|
case *declaration.Archive:
|
||||||
|
if res.Source != "" {
|
||||||
|
fetched[res.Digest] = true
|
||||||
|
}
|
||||||
|
case *declaration.Process:
|
||||||
|
// A runtime of its own runs the tool runner's program, which the tool runner's want checks (A-432).
|
||||||
|
if res.Source != "" && !strings.HasSuffix(res.ID, "."+catalogue.OwnRuntimeID()) {
|
||||||
|
fetched[res.Digest] = true
|
||||||
|
}
|
||||||
|
case *declaration.Container:
|
||||||
|
// A digest the want's parts name always counts as fetched, whatever marks the resource carries; one they do not
|
||||||
|
// name counts only when it is the mesh's own: an image the manifest pins from outside, or the registry seat's
|
||||||
|
// copy of an upstream image, is no part (design 47 G A-454, A-454a).
|
||||||
|
digest := ""
|
||||||
|
if strings.HasPrefix(res.Image, "sha256:") {
|
||||||
|
digest = res.Image
|
||||||
|
} else if _, d, ok := strings.Cut(res.Image, "@"); ok {
|
||||||
|
digest = d
|
||||||
|
}
|
||||||
|
if digest == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !inParts[digest] && (res.ImageSeat == "" || upstreamCopy(res.Image)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fetched[digest] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
named := map[string]bool{}
|
||||||
|
for _, d := range parts {
|
||||||
|
named[d] = true
|
||||||
|
}
|
||||||
|
var outside, missing []string
|
||||||
|
for d := range fetched {
|
||||||
|
if !named[d] {
|
||||||
|
outside = append(outside, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for d := range named {
|
||||||
|
if !fetched[d] {
|
||||||
|
missing = append(missing, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(outside) == 0 && len(missing) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
sort.Strings(outside)
|
||||||
|
sort.Strings(missing)
|
||||||
|
var said []string
|
||||||
|
if len(outside) > 0 {
|
||||||
|
said = append(said, "fetched and not in the want's parts: "+strings.Join(outside, ", "))
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
said = append(said, "in the want's parts and not fetched: "+strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %s", PartsDiffer, strings.Join(said, "; "))
|
||||||
|
}
|
||||||
+349
-20
@@ -312,11 +312,54 @@ type Scope struct {
|
|||||||
Module string
|
Module string
|
||||||
// Others are the resources of every other module the node has a composition for.
|
// Others are the resources of every other module the node has a composition for.
|
||||||
Others []declaration.Resource
|
Others []declaration.Resource
|
||||||
|
// Placed are the modules a want still places on the node (present, not removed), held ones included: no apply but
|
||||||
|
// the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node,
|
||||||
|
// composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing.
|
||||||
|
Placed map[string]bool
|
||||||
|
// RemovedByWant are the modules whose want says removed. In the node's own apply (Module NodeModule) with this set
|
||||||
|
// non-nil, a resource of any other module is removed only when that module is here: a module the node runs folded
|
||||||
|
// in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on
|
||||||
|
// the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone.
|
||||||
|
RemovedByWant map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own:
|
||||||
|
// the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded
|
||||||
|
// module is recorded as the node's, its id naming the module). Empty when there is none.
|
||||||
|
func (sc *Scope) placedOwner(a store.Applied) string {
|
||||||
|
if sc == nil || (len(sc.Placed) == 0 && sc.RemovedByWant == nil) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
m := a.Module
|
||||||
|
if m == "" || m == store.NodeModule {
|
||||||
|
m, _ = declaration.ModuleGuess(a.ID)
|
||||||
|
}
|
||||||
|
if m == "" || m == sc.Module || m == store.NodeModule {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if sc.Placed[m] {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
if sc.Module == store.NodeModule && sc.RemovedByWant != nil && !sc.RemovedByWant[m] {
|
||||||
|
return m // a module the node's own composition carried, which no want removed
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// owns reports whether a record belongs to the scope's module.
|
// owns reports whether a record belongs to the scope's module.
|
||||||
|
//
|
||||||
|
// In a module's own apply, a record the node's own composition made for the module while it was folded (recorded as the
|
||||||
|
// node's, its id naming the module) is the module's too: it is the only apply that may remove it once neither
|
||||||
|
// composition declares it (issue 774). One the node's own still declares stays, as another's held resource does.
|
||||||
func (sc *Scope) owns(a store.Applied) bool {
|
func (sc *Scope) owns(a store.Applied) bool {
|
||||||
return store.OwnerOf(a, declaration.ModuleGuess) == sc.Module
|
if store.OwnerOf(a, declaration.ModuleGuess) == sc.Module {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if sc.Module == store.NodeModule || a.Module != store.NodeModule {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
m, ok := declaration.ModuleGuess(a.ID)
|
||||||
|
return ok && m == sc.Module
|
||||||
}
|
}
|
||||||
|
|
||||||
// isNode is whether the scope is the node's own composition, the one that carries identities, groups, libraries and
|
// isNode is whether the scope is the node's own composition, the one that carries identities, groups, libraries and
|
||||||
@@ -441,6 +484,57 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A resource the node's own carried for a module that now has a composition of its own is that module's to remove**
|
||||||
|
// (issue 774, 2026-10-30 on the bus node): recorded as the node's while the module was folded, it is no longer in the
|
||||||
|
// node's own composition and no longer in the module's own either (restic's measure, replaced by awake). The node's own
|
||||||
|
// apply leaves it, record and all, and says so; the module's own apply sees it as its own (Scope.owns) and removes it.
|
||||||
|
// Refused instead, the node's own applied nothing at all, every two minutes, for as long as the record stayed.
|
||||||
|
takenByItsModule := map[string]string{}
|
||||||
|
if scope != nil && scope.Module == store.NodeModule {
|
||||||
|
composes := map[string]bool{}
|
||||||
|
for _, r := range scope.Others {
|
||||||
|
if m := r.DeclaredModule(); m != "" && m != store.NodeModule {
|
||||||
|
composes[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, orphan := range known.Orphans(declared, origin) {
|
||||||
|
if orphan.Module != store.NodeModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m, ok := declaration.ModuleGuess(orphan.ID); ok && composes[m] {
|
||||||
|
takenByItsModule[orphan.ID] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module a want still places here is never removed by another's apply** (issue 774): its resources go only with
|
||||||
|
// its own apply or its want's removal. One that would is refused whole, before anything is changed, naming them: a
|
||||||
|
// composition that leaves them out lost what runs, and removing what runs is the worst a node can do with that.
|
||||||
|
if scope != nil {
|
||||||
|
var refused []string
|
||||||
|
for _, orphan := range known.Orphans(declared, origin) {
|
||||||
|
if !scope.owns(orphan) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, kept := heldByOthers[orphan.ID]; kept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, kept := takenByItsModule[orphan.ID]; kept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m := scope.placedOwner(orphan); m != "" {
|
||||||
|
refused = append(refused, orphan.ID+" of "+m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(refused) > 0 {
|
||||||
|
sort.Strings(refused)
|
||||||
|
err := fmt.Errorf("the apply of %s would remove %s, of modules a want still places here; a module's resources "+
|
||||||
|
"go only with its own apply or its want's removal, so nothing was applied", scope.Module, strings.Join(refused, ", "))
|
||||||
|
log(" refused: " + err.Error())
|
||||||
|
return report, known, &Error{Resource: "the resources of modules placed here", Err: err, Done: report}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
removeOrphan := func(orphan store.Applied) error {
|
removeOrphan := func(orphan store.Applied) error {
|
||||||
var action, detail string
|
var action, detail string
|
||||||
var err error
|
var err error
|
||||||
@@ -456,6 +550,16 @@ func applyScoped(
|
|||||||
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if module, taken := takenByItsModule[orphan.ID]; taken {
|
||||||
|
detail := "no longer declared here; " + module + " has a composition of its own here, whose apply removes it, " +
|
||||||
|
"so it was left as it is"
|
||||||
|
report.Outcomes = append(report.Outcomes, Outcome{
|
||||||
|
ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
|
||||||
|
Action: "kept", Detail: detail,
|
||||||
|
})
|
||||||
|
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if by, held := heldFiles[fileKey(orphan)]; held && by != orphan.ID {
|
if by, held := heldFiles[fileKey(orphan)]; held && by != orphan.ID {
|
||||||
if scope != nil && !declared[by] {
|
if scope != nil && !declared[by] {
|
||||||
// Another module declares the path: it is that module's, so this record is let go and the file is
|
// Another module declares the path: it is that module's, so this record is let go and the file is
|
||||||
@@ -474,6 +578,17 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
if declaration.Type(orphan.Type) == declaration.TypeService {
|
if declaration.Type(orphan.Type) == declaration.TypeService {
|
||||||
if by, held := heldUnits[unitKey(orphan.Scope, orphan.User, orphan.Target)]; held {
|
if by, held := heldUnits[unitKey(orphan.Scope, orphan.User, orphan.Target)]; held {
|
||||||
|
// **What the going record found passes to the record that stays** (design 42 zfs-owns-zfs §2, answer
|
||||||
|
// A-21), as a file is handed to the resource still declaring its path (ADR 0223): the unit as the machine
|
||||||
|
// had it before the mesh is what the staying record gives back when it goes, not what the mesh made of
|
||||||
|
// it. Only where the staying record has no finding of its own, kept as an unfinished apply's is.
|
||||||
|
if orphan.Found != nil && declared[by] {
|
||||||
|
if rec, has := known.Find(by); !has || rec.Found == nil {
|
||||||
|
if _, pending := known.FoundFirst[by]; !pending {
|
||||||
|
known.KeepFound(by, origin, *orphan.Found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
known.Forget(orphan.ID)
|
known.Forget(orphan.ID)
|
||||||
detail := "no longer declared; " + by + " still holds the unit, so it was left as it is"
|
detail := "no longer declared; " + by + " still holds the unit, so it was left as it is"
|
||||||
report.Outcomes = append(report.Outcomes, Outcome{
|
report.Outcomes = append(report.Outcomes, Outcome{
|
||||||
@@ -506,7 +621,7 @@ func applyScoped(
|
|||||||
err = fmt.Errorf("%w: %s", errRemovalWaits, said)
|
err = fmt.Errorf("%w: %s", errRemovalWaits, said)
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
action, detail, err = remove(ctx, sys, orphan, run, made)
|
action, detail, err = remove(ctx, sys, orphan, run, made, journalDir(windows))
|
||||||
}
|
}
|
||||||
if errors.Is(err, errNoRemoval) && store.IsFormer(orphan.ID) {
|
if errors.Is(err, errNoRemoval) && store.IsFormer(orphan.ID) {
|
||||||
// **A former target of a kind the host cannot remove is left in place and forgotten,
|
// **A former target of a kind the host cannot remove is left in place and forgotten,
|
||||||
@@ -604,10 +719,24 @@ func applyScoped(
|
|||||||
ordered = append(afterWhatTheyRead(guard), afterWhatTheyRead(rest)...)
|
ordered = append(afterWhatTheyRead(guard), afterWhatTheyRead(rest)...)
|
||||||
guardFirst = len(guard)
|
guardFirst = len(guard)
|
||||||
}
|
}
|
||||||
|
// Modules whose own step did not complete. What follows *within such a module* is not attempted;
|
||||||
|
// the rest of the machine is (novox/hq ADR 0136).
|
||||||
|
gated := map[string]bool{}
|
||||||
|
// Why each gated module was gated, said on every resource of it that was skipped.
|
||||||
|
gatedWhy := map[string]string{}
|
||||||
orphansRemoved := false
|
orphansRemoved := false
|
||||||
removeOrphans := func() error {
|
removeOrphans := func() error {
|
||||||
orphansRemoved = true
|
orphansRemoved = true
|
||||||
for _, orphan := range orphans {
|
for _, orphan := range orphans {
|
||||||
|
// **Nothing of a module whose image or artifact could not be had is removed either** (novox/hq issue 1966):
|
||||||
|
// a removal is as much a change of what it runs as a replacement, and the module keeps running what it ran.
|
||||||
|
if m := orphanModule(orphan); m != "" && gated[m] {
|
||||||
|
report.Outcomes = append(report.Outcomes, Outcome{
|
||||||
|
ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "kept", Detail: gatedWhy[m],
|
||||||
|
})
|
||||||
|
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, gatedWhy[m]))
|
||||||
|
continue
|
||||||
|
}
|
||||||
if err := removeOrphan(orphan); err != nil {
|
if err := removeOrphan(orphan); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -656,7 +785,7 @@ func applyScoped(
|
|||||||
// change one apply late, and never misses it.
|
// change one apply late, and never misses it.
|
||||||
// And one patience with the artifact store for the whole apply (novox/hq issue 291): a fetch it
|
// And one patience with the artifact store for the whole apply (novox/hq issue 291): a fetch it
|
||||||
// does not answer during a maintenance window waits for the window instead of failing.
|
// does not answer during a maintenance window waits for the window instead of failing.
|
||||||
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
||||||
groups: wanted, handed: handed}
|
groups: wanted, handed: handed}
|
||||||
// Where nothing is placed or mounted from, judged before anything is applied (novox/hq issue 339), and
|
// Where nothing is placed or mounted from, judged before anything is applied (novox/hq issue 339), and
|
||||||
// which declared directories were already here: one this apply makes is the mesh's, one found may not be.
|
// which declared directories were already here: one this apply makes is the mesh's, one found may not be.
|
||||||
@@ -693,11 +822,6 @@ func applyScoped(
|
|||||||
var failures []*Error
|
var failures []*Error
|
||||||
// Which resources applied without error in this run, for the move-out pass after the loop (issue 1024).
|
// Which resources applied without error in this run, for the move-out pass after the loop (issue 1024).
|
||||||
appliedOK := map[string]bool{}
|
appliedOK := map[string]bool{}
|
||||||
// Modules whose own step did not complete. What follows *within such a module* is not attempted;
|
|
||||||
// the rest of the machine is (novox/hq ADR 0136).
|
|
||||||
gated := map[string]bool{}
|
|
||||||
// Why each gated module was gated, said on every resource of it that was skipped.
|
|
||||||
gatedWhy := map[string]string{}
|
|
||||||
gatedModules := func() []string {
|
gatedModules := func() []string {
|
||||||
out := make([]string, 0, len(gated))
|
out := make([]string, 0, len(gated))
|
||||||
for m := range gated {
|
for m := range gated {
|
||||||
@@ -835,6 +959,18 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
setRunnerWarnings(warnings)
|
setRunnerWarnings(warnings)
|
||||||
|
// **Every new image of a module is on the machine before any of its containers is set aside** (design 51, issue 1966
|
||||||
|
// ENG-7). A container replaced after the first one's turn pulled its image only then, and for a module whose first
|
||||||
|
// container serves the store its second image comes from, that pull met the stopped one. A pull that fails
|
||||||
|
// replaces nothing of the module: its containers keep running what they ran, the failure names the image, and what
|
||||||
|
// follows in the module is skipped as it is for any step that did not complete.
|
||||||
|
in.pulled = map[string]bool{}
|
||||||
|
pullFirst(ctx, ordered, d, in.pulled, run, &failures, gated, gatedWhy, log)
|
||||||
|
// And every archive and process, fetched before anything is replaced (issue 1966). Its failures are counted with the
|
||||||
|
// others once the resources are through, as an identity's are: here, before the guard, they would read as the guard
|
||||||
|
// not being up, and nothing of the machine would be applied.
|
||||||
|
var fetchFailures []*Error
|
||||||
|
fetchFirst(ctx, ordered, d, &in, &fetchFailures, gated, gatedWhy, log)
|
||||||
for i, resource := range ordered {
|
for i, resource := range ordered {
|
||||||
if !orphansRemoved && i == guardFirst {
|
if !orphansRemoved && i == guardFirst {
|
||||||
// **Only a guard that is up may let the filter go.** Removing the derived filter's
|
// **Only a guard that is up may let the filter go.** Removing the derived filter's
|
||||||
@@ -1037,6 +1173,16 @@ func applyScoped(
|
|||||||
if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
|
if p, ok := resource.(*declaration.Process); ok && p.RunOnce {
|
||||||
gates = true
|
gates = true
|
||||||
}
|
}
|
||||||
|
// **A module's apply stops at its first failed resource** (design 42 zfs-owns-zfs §1, ADR 0679 §16 (a)): any
|
||||||
|
// other resource of a module that failed gates the rest of that module, which is reported skipped with this
|
||||||
|
// reason. What it applied before stays applied; other modules, and a resource of no module, go on as before.
|
||||||
|
if module, ours := d.ModuleOfID(resource.Identity()); ours && !gates {
|
||||||
|
failed.Gated = true
|
||||||
|
gated[module] = true
|
||||||
|
gatedWhy[module] = fmt.Sprintf("%s failed, so nothing more of this module was attempted", resource.Identity())
|
||||||
|
log(fmt.Sprintf(" gated %s.*: %s failed, so the rest of it was not attempted", module, resource.Identity()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
if gates {
|
if gates {
|
||||||
failed.Gated = true
|
failed.Gated = true
|
||||||
// **A step that failed has not completed, whatever it did before** (novox/hq issue 467). A
|
// **A step that failed has not completed, whatever it did before** (novox/hq issue 467). A
|
||||||
@@ -1193,6 +1339,7 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
|
|
||||||
failures = append(failures, identityFailures...)
|
failures = append(failures, identityFailures...)
|
||||||
|
failures = append(failures, fetchFailures...)
|
||||||
|
|
||||||
// A directory or archive that says move-out keeps only what it knows (issue 1024, novox/hq ADR 0422).
|
// A directory or archive that says move-out keeps only what it knows (issue 1024, novox/hq ADR 0422).
|
||||||
movedOut, leftIn, moveFailures := moveOutPass(d, known, appliedOK, log)
|
movedOut, leftIn, moveFailures := moveOutPass(d, known, appliedOK, log)
|
||||||
@@ -1223,6 +1370,19 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A file written or removed in a unit directory is read by the service manager only after a reload, whether or not
|
||||||
|
// a service reflects it: a drop-in on reboot.target is reflected by no service, and without the reload the next
|
||||||
|
// reboot still runs the target as it was loaded (issue 1126). Once per apply.
|
||||||
|
if at := unitFileMoved(report.Outcomes); at != "" {
|
||||||
|
if u, ok := sys.(unitReloader); ok {
|
||||||
|
if err := u.ReloadUnits(ctx, run); err != nil {
|
||||||
|
failures = append(failures, &Error{Resource: at, Err: fmt.Errorf("reloading the service manager's units: %w", err)})
|
||||||
|
} else {
|
||||||
|
log(fmt.Sprintf(" reloaded the service manager's units: %s changed", at))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
|
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
|
||||||
if origin == store.OriginDeclared && scope.isNode() {
|
if origin == store.OriginDeclared && scope.isNode() {
|
||||||
// And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328
|
// And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328
|
||||||
@@ -1455,9 +1615,17 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
|||||||
case *declaration.User:
|
case *declaration.User:
|
||||||
return applyUser(ctx, sys, res, run, previous, in.groups, in.handed)
|
return applyUser(ctx, sys, res, run, previous, in.groups, in.handed)
|
||||||
case *declaration.Archive:
|
case *declaration.Archive:
|
||||||
return applyArchive(ctx, res, previous, in.away)
|
held, err := in.artifact(res.Identity())
|
||||||
|
if err != nil {
|
||||||
|
return begin(r), err
|
||||||
|
}
|
||||||
|
return applyArchive(ctx, res, previous, held, in.away, journalDir(in.windows))
|
||||||
case *declaration.Process:
|
case *declaration.Process:
|
||||||
return applyProcess(ctx, res, run, changed, previous, in.away)
|
held, err := in.artifact(res.Identity())
|
||||||
|
if err != nil {
|
||||||
|
return begin(r), err
|
||||||
|
}
|
||||||
|
return applyProcess(ctx, withNode(res, in.node), run, changed, previous, held, in.away)
|
||||||
case *declaration.Action:
|
case *declaration.Action:
|
||||||
return applyAction(ctx, res, run)
|
return applyAction(ctx, res, run)
|
||||||
case *declaration.Network:
|
case *declaration.Network:
|
||||||
@@ -1997,11 +2165,68 @@ type serviceReloader interface {
|
|||||||
ReloadService(ctx context.Context, run system.Runner, unit string) error
|
ReloadService(ctx context.Context, run system.Runner, unit string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UnitDirs are the system service manager's unit directories: a file the apply writes or removes in one of them, or in a
|
||||||
|
// drop-in directory below one, is read by the manager only after a reload (issue 1126). Replaceable in tests.
|
||||||
|
var UnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/usr/local/lib/systemd/system"}
|
||||||
|
|
||||||
|
// unitFileMoved is the first file this apply wrote or removed in a unit directory or a drop-in directory below one,
|
||||||
|
// or "" when it moved none.
|
||||||
|
func unitFileMoved(outcomes []Outcome) string {
|
||||||
|
for _, o := range outcomes {
|
||||||
|
// Only what changed the file on disk, as MovesRoot reads it: skipped, held, kept, forgotten and waiting come
|
||||||
|
// back on every apply and change nothing.
|
||||||
|
if declaration.Type(o.Type) != declaration.TypeFile {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch o.Action {
|
||||||
|
case "created", "updated", "corrected", "removed":
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := filepath.Clean(o.Target)
|
||||||
|
for _, dir := range UnitDirs {
|
||||||
|
if rel, err := filepath.Rel(dir, at); err == nil && rel != "." && !strings.HasPrefix(rel, "..") {
|
||||||
|
return at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// unitReloader is a service manager that caches unit files and must be told to read them again.
|
// unitReloader is a service manager that caches unit files and must be told to read them again.
|
||||||
type unitReloader interface {
|
type unitReloader interface {
|
||||||
ReloadUnits(ctx context.Context, run system.Runner) error
|
ReloadUnits(ctx context.Context, run system.Runner) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restReader is a service manager that can say whether a one-shot has nothing running now
|
||||||
|
// (novox/hq issue 1126).
|
||||||
|
type restReader interface {
|
||||||
|
ServiceAtRest(ctx context.Context, run system.Runner, unit string) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredState is a unit's state as a declaration of want reads it. **A one-shot at rest is
|
||||||
|
// stopped to a declaration that it be stopped** (novox/hq issue 1126): a one-shot that does not
|
||||||
|
// linger reads "running" once it ran and worked — that is what a declaration that it run asks — but
|
||||||
|
// it has no process between its runs, so it is also as stopped as it can be, and stopping it changes
|
||||||
|
// nothing. Read only as "running", a boot-time cleanup declared stopped and enabled failed every
|
||||||
|
// apply after the boot that ran it: "was asked to be stopped and is running".
|
||||||
|
func declaredState(ctx context.Context, sys system.System, run Runner, unit, want string) (string, error) {
|
||||||
|
state, err := sys.ServiceState(ctx, run, unit)
|
||||||
|
if err != nil || want != "stopped" || state != "running" {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
if r, ok := sys.(restReader); ok {
|
||||||
|
rest, err := r.ServiceAtRest(ctx, run, unit)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if rest {
|
||||||
|
return "stopped", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
|
// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
|
||||||
// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
|
// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
|
||||||
var serviceSettle = 2 * time.Second
|
var serviceSettle = 2 * time.Second
|
||||||
@@ -2154,7 +2379,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
|||||||
// own stopping from anybody else's (novox/hq ADR 0269).
|
// own stopping from anybody else's (novox/hq ADR 0269).
|
||||||
out.declared = &store.FoundUnit{Unit: r.Unit, State: r.State, Boot: r.Boot}
|
out.declared = &store.FoundUnit{Unit: r.Unit, State: r.State, Boot: r.Boot}
|
||||||
|
|
||||||
before, err := sys.ServiceState(ctx, run, r.Unit)
|
before, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
@@ -2165,7 +2390,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
|||||||
|
|
||||||
// Read back. A service manager accepting a command says the transaction was accepted,
|
// Read back. A service manager accepting a command says the transaction was accepted,
|
||||||
// not that the unit is running — one that starts and immediately dies satisfies it.
|
// not that the unit is running — one that starts and immediately dies satisfies it.
|
||||||
after, err := sys.ServiceState(ctx, run, r.Unit)
|
after, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||||
if err == nil && r.State == "running" {
|
if err == nil && r.State == "running" {
|
||||||
after, err = stayedRunning(ctx, sys, run, r.Unit)
|
after, err = stayedRunning(ctx, sys, run, r.Unit)
|
||||||
}
|
}
|
||||||
@@ -2309,7 +2534,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
|
|||||||
//
|
//
|
||||||
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
|
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
|
||||||
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||||
made map[string]bool) (string, string, error) {
|
made map[string]bool, journals string) (string, string, error) {
|
||||||
switch declaration.Type(a.Type) {
|
switch declaration.Type(a.Type) {
|
||||||
case declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive:
|
case declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive:
|
||||||
// Removal below a home follows no link an account put there either (novox/hq ADR 0266).
|
// Removal below a home follows no link an account put there either (novox/hq ADR 0266).
|
||||||
@@ -2435,7 +2660,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
|||||||
case declaration.TypeArchive:
|
case declaration.TypeArchive:
|
||||||
// Exactly what it unpacked, and the directories the host made for it once they are empty
|
// Exactly what it unpacked, and the directories the host made for it once they are empty
|
||||||
// (novox/hq issue 162).
|
// (novox/hq issue 162).
|
||||||
return removeArchive(a)
|
return removeArchive(a, journals)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return "", "", fmt.Errorf("%w: a %q", errNoRemoval, a.Type)
|
return "", "", fmt.Errorf("%w: a %q", errNoRemoval, a.Type)
|
||||||
@@ -2585,6 +2810,15 @@ type inputs struct {
|
|||||||
files map[string]bool
|
files map[string]bool
|
||||||
// handed is every group a record that went this apply handed to a resource still asking for it.
|
// handed is every group a record that went this apply handed to a resource still asking for it.
|
||||||
handed Handed
|
handed Handed
|
||||||
|
// pulled is every image this apply found on the machine, or pulled, before any container was replaced (pullFirst):
|
||||||
|
// applyContainer does not ask for it again.
|
||||||
|
pulled map[string]bool
|
||||||
|
// fetched is every archive's and process's bytes this apply fetched before anything was replaced (fetchFirst), each
|
||||||
|
// taken by its applier once (artifact).
|
||||||
|
fetched map[string]fetchedArtifact
|
||||||
|
// node is the node's own name, the declaration's For: every process runs with it as MESH_NODE beside its own env
|
||||||
|
// (design 47 G A-342). Empty where the declaration names none (the first node's carried bundle).
|
||||||
|
node string
|
||||||
}
|
}
|
||||||
|
|
||||||
// fileDigest is what a file the container reads holds, by digest.
|
// fileDigest is what a file the container reads holds, by digest.
|
||||||
@@ -2991,8 +3225,10 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
|||||||
// gone: an image the registry does not hold then left the service down and every later apply
|
// gone: an image the registry does not hold then left the service down and every later apply
|
||||||
// failing. Pulled (or found present) first, an unobtainable image fails this resource loudly,
|
// failing. Pulled (or found present) first, an unobtainable image fails this resource loudly,
|
||||||
// naming the image, and the old container keeps running.
|
// naming the image, and the old container keeps running.
|
||||||
if err := ensureImage(ctx, cri, r.Image, run); err != nil {
|
if !in.pulled[r.Image] {
|
||||||
return out, fmt.Errorf("not replacing container %s, which keeps running: %w", r.Name, err)
|
if err := ensureImage(ctx, cri, r.Image, run); err != nil {
|
||||||
|
return out, fmt.Errorf("not replacing container %s, which keeps running: %w", r.Name, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// **The running container is set aside, never removed, until its replacement runs** (issue 1945): stopped and
|
// **The running container is set aside, never removed, until its replacement runs** (issue 1945): stopped and
|
||||||
// renamed, so its name and ports are free, and started again under its own name when the replacement does not
|
// renamed, so its name and ports are free, and started again under its own name when the replacement does not
|
||||||
@@ -3295,16 +3531,17 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error {
|
|||||||
// already hold — built here, or handed over. There is no registry that answers for it, so
|
// already hold — built here, or handed over. There is no registry that answers for it, so
|
||||||
// pulling would fail somewhere that names a network problem instead of a missing image.
|
// pulling would fail somewhere that names a network problem instead of a missing image.
|
||||||
if strings.HasPrefix(image, "sha256:") {
|
if strings.HasPrefix(image, "sha256:") {
|
||||||
return fmt.Errorf(
|
return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf(
|
||||||
"image %s is not on this machine, and an image named by its own digest cannot be "+
|
"image %s is not on this machine, and an image named by its own digest cannot be "+
|
||||||
"fetched: nothing serves it. Build it here, or load it, before applying this",
|
"fetched: nothing serves it. Build it here, or load it, before applying this",
|
||||||
image)
|
image)}
|
||||||
}
|
}
|
||||||
|
started := time.Now()
|
||||||
if _, err := run(ctx, cri, "pull", image); err != nil {
|
if _, err := run(ctx, cri, "pull", image); err != nil {
|
||||||
return fmt.Errorf("pulling image %s: %w", image, err)
|
return &PullError{Image: image, Started: started, Err: fmt.Errorf("pulling image %s: %w", image, err)}
|
||||||
}
|
}
|
||||||
if _, err := run(ctx, cri, "image", "inspect", image); err != nil {
|
if _, err := run(ctx, cri, "image", "inspect", image); err != nil {
|
||||||
return fmt.Errorf("image %s is not present after pulling it: %w", image, err)
|
return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf("image %s is not present after pulling it: %w", image, err)}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -4166,3 +4403,95 @@ func ownerText(info os.FileInfo) string {
|
|||||||
}
|
}
|
||||||
return fmt.Sprintf("%d:%d", uid, gid)
|
return fmt.Sprintf("%d:%d", uid, gid)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// orphanModule is the module a record going belongs to, by its id, else the module whose want it was applied for.
|
||||||
|
func orphanModule(a store.Applied) string {
|
||||||
|
if m, ok := declaration.ModuleGuess(a.ID); ok {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
return a.Module
|
||||||
|
}
|
||||||
|
|
||||||
|
// pullFirst finds on the machine, or pulls, the image of each container the declaration runs before any is replaced
|
||||||
|
// (ENG-7) for a module of two containers or more. A container that is run once is left to its marker, and a runtime that
|
||||||
|
// does not answer to applyContainer, which says so. The first image that cannot be had fails its container with the image named and gates its module, so none of
|
||||||
|
// its containers is touched; the images found before it stay pulled.
|
||||||
|
func pullFirst(ctx context.Context, ordered []declaration.Resource, d *declaration.Declaration, pulled map[string]bool,
|
||||||
|
run Runner, failures *[]*Error, gated map[string]bool, gatedWhy map[string]string, log func(string)) {
|
||||||
|
// A module of one container already pulls its image before it sets that container aside (applyContainer), and nothing
|
||||||
|
// of it comes before: only a module of two or more has an image pulled after one of its containers was replaced.
|
||||||
|
count := map[string]int{}
|
||||||
|
for _, resource := range ordered {
|
||||||
|
if c, isContainer := resource.(*declaration.Container); isContainer && c.Image != "" && !c.RunOnce && c.Schedule == "" {
|
||||||
|
module, _ := d.ModuleOfID(c.Identity())
|
||||||
|
count[module]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var cri string
|
||||||
|
for _, resource := range ordered {
|
||||||
|
c, isContainer := resource.(*declaration.Container)
|
||||||
|
if !isContainer || c.Image == "" || c.RunOnce || c.Schedule != "" || pulled[c.Image] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, ours := d.ModuleOfID(c.Identity())
|
||||||
|
if count[module] < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ours && gated[module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if cri == "" {
|
||||||
|
var err error
|
||||||
|
if cri, err = containerRuntime(ctx, run); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ensureImage(ctx, cri, c.Image, run); err != nil {
|
||||||
|
err = fmt.Errorf("not replacing container %s, or any other container of its module, which keep running: %w", c.Name, err)
|
||||||
|
*failures = append(*failures, &Error{Resource: c.Identity(), Err: err})
|
||||||
|
log(fmt.Sprintf(" failed %s (%s): %v", c.Identity(), c.Target(), err))
|
||||||
|
if ours {
|
||||||
|
gated[module] = true
|
||||||
|
gatedWhy[module] = "its image " + c.Image + " could not be had, so none of its containers was replaced"
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pulled[c.Image] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullError is an image that could not be found or pulled, as ensureImage fails: what the node agent reads to tell a failed
|
||||||
|
// pull from another failed apply (ENG-8).
|
||||||
|
type PullError struct {
|
||||||
|
Image string
|
||||||
|
// Started is when the pull began, which the node agent compares with when a store's holder settled (ENG-11).
|
||||||
|
Started time.Time
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullErrors are the pull errors of every failure an apply reports, the first among them and the rest in Failures, in the
|
||||||
|
// order they happened: what the node agent tells a failed pull from another failed apply by (ENG-8). Nil for an error
|
||||||
|
// that is no apply's, or whose failures met none.
|
||||||
|
func PullErrors(err error) []*PullError {
|
||||||
|
var out []*PullError
|
||||||
|
var e *Error
|
||||||
|
if errors.As(err, &e) {
|
||||||
|
for _, f := range e.Failures {
|
||||||
|
var p *PullError
|
||||||
|
if errors.As(f.Err, &p) {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out) > 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var p *PullError
|
||||||
|
if errors.As(err, &p) {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *PullError) Error() string { return e.Err.Error() }
|
||||||
|
func (e *PullError) Unwrap() error { return e.Err }
|
||||||
|
|||||||
+162
-64
@@ -182,77 +182,93 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) {
|
// A failed resource fails the apply and names itself, and what else is attempted depends on whose it is
|
||||||
// The apply fails, names the resource, and carries what did happen — because the machine is
|
// (novox/hq 04-ISSUES/011; design 42 zfs-owns-zfs §1, ADR 0679 §16 (a)): the resources of no module are all
|
||||||
// in whatever state the apply reached and the only honest thing to hand back is that list.
|
// attempted, so one broken resource holds no machine hostage and every failure is counted, not just the first;
|
||||||
//
|
// a module's apply stops at its first failed resource, reports each one left, and no other module is touched.
|
||||||
// **And everything is attempted.** It used to stop at the first failure, which made one
|
func TestAFailedResourceFailsTheApplyAndWhatFollowsIsAttemptedByWhoseItIs(t *testing.T) {
|
||||||
// broken resource hold the whole machine hostage: a module declaring a package that does not
|
|
||||||
// exist meant every module after it was never applied, for ever
|
|
||||||
// (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an
|
|
||||||
// earlier one — and it still may, and it then fails its own check and is reported, which is
|
|
||||||
// more information than not attempting it.
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
blocker := filepath.Join(dir, "blocker")
|
for _, name := range []string{"blocker", "one", "two"} {
|
||||||
if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("i am a file\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d := parse(t, `{"declaration":1,"resources":[
|
|
||||||
{"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"},
|
|
||||||
{"id":"doomed","type":"directory","path":"`+blocker+`"},
|
|
||||||
{"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"}
|
|
||||||
]}`)
|
|
||||||
|
|
||||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("an impossible resource did not fail the apply")
|
|
||||||
}
|
|
||||||
|
|
||||||
var applyErr *Error
|
|
||||||
if !errors.As(err, &applyErr) {
|
|
||||||
t.Fatalf("expected an apply error, got %T", err)
|
|
||||||
}
|
|
||||||
if applyErr.Resource != "doomed" {
|
|
||||||
t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource)
|
|
||||||
}
|
|
||||||
// Everything that worked is in the report, before and after the failure.
|
|
||||||
if len(applyErr.Done.Outcomes) != 2 {
|
|
||||||
t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil {
|
|
||||||
t.Error("a resource after the failing one was never attempted, so one broken module " +
|
|
||||||
"still blocks every module after it")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) {
|
|
||||||
// "One thing failed" and "eleven things failed" are different machines, and the first line is
|
|
||||||
// what somebody reads.
|
|
||||||
dir := t.TempDir()
|
|
||||||
for _, name := range []string{"one", "two"} {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
d := parse(t, `{"declaration":1,"resources":[
|
file := func(id, name string) string {
|
||||||
{"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"},
|
return `{"id":"` + id + `","type":"file","path":"` + filepath.Join(dir, name) + `","content":"a\n"}`
|
||||||
{"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"}
|
|
||||||
]}`)
|
|
||||||
|
|
||||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("two impossible resources did not fail the apply")
|
|
||||||
}
|
}
|
||||||
var applyErr *Error
|
unwritable := func(id, name string) string { // a file below a file: it cannot be made
|
||||||
if !errors.As(err, &applyErr) {
|
return `{"id":"` + id + `","type":"file","path":"` + filepath.Join(dir, name, "x.conf") + `","content":"a\n"}`
|
||||||
t.Fatalf("got %T", err)
|
|
||||||
}
|
}
|
||||||
if applyErr.Others != 1 {
|
doomed := func(id, name string) string {
|
||||||
t.Errorf("the failure says %d others also failed, and one did", applyErr.Others)
|
return `{"id":"` + id + `","type":"directory","path":"` + filepath.Join(dir, name) + `"}`
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "one other resource also failed") {
|
exists := func(name string) bool {
|
||||||
t.Errorf("the message does not say others failed: %v", err)
|
_, err := os.Stat(filepath.Join(dir, name))
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
resources []string
|
||||||
|
failed string
|
||||||
|
others int
|
||||||
|
applied []string // files that must exist
|
||||||
|
left []string // files that must not, reported skipped
|
||||||
|
detail string // what the module's result says, when it is one module's
|
||||||
|
}{
|
||||||
|
{"of no module, the rest is attempted", []string{file("fine", "fine.conf"), doomed("doomed", "blocker"), file("after", "after.conf")},
|
||||||
|
"doomed", 0, []string{"fine.conf", "after.conf"}, nil, ""},
|
||||||
|
{"of no module, every failure is counted", []string{doomed("first", "one"), doomed("second", "two")},
|
||||||
|
"first", 1, nil, nil, ""},
|
||||||
|
{"of a module, it stops there and names each one left",
|
||||||
|
[]string{file("m.first", "first.conf"), unwritable("m.second", "blocker"), file("m.third", "third.conf"), file("m.fourth", "fourth.conf"), file("o.other", "other.conf")},
|
||||||
|
"m.second", 0, []string{"first.conf", "other.conf"}, []string{"third.conf", "fourth.conf"},
|
||||||
|
"m.second failed"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[`+strings.Join(tc.resources, ",")+`]}`)
|
||||||
|
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
|
||||||
|
var applyErr *Error
|
||||||
|
if !errors.As(err, &applyErr) {
|
||||||
|
t.Fatalf("expected an apply error, got %v", err)
|
||||||
|
}
|
||||||
|
if applyErr.Resource != tc.failed || applyErr.Others != tc.others {
|
||||||
|
t.Errorf("the failure names %q with %d others, want %q with %d", applyErr.Resource, applyErr.Others, tc.failed, tc.others)
|
||||||
|
}
|
||||||
|
if tc.others == 1 && !strings.Contains(err.Error(), "one other resource also failed") {
|
||||||
|
t.Errorf("the message does not say others failed: %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range tc.applied {
|
||||||
|
if !exists(name) {
|
||||||
|
t.Errorf("%s was not applied", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range tc.left {
|
||||||
|
if exists(name) {
|
||||||
|
t.Errorf("%s was applied after its module failed", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.detail == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
outcomes := applyErr.Done.Outcomes
|
||||||
|
skipped := 0
|
||||||
|
for _, o := range outcomes {
|
||||||
|
if o.Action == "skipped" {
|
||||||
|
skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if skipped != len(tc.left) {
|
||||||
|
t.Errorf("%d resources reported skipped, want %d: %+v", skipped, len(tc.left), outcomes)
|
||||||
|
}
|
||||||
|
result := ModuleResults(d, report, err)["m"]
|
||||||
|
if !result.Failed || !strings.Contains(result.Detail, tc.detail) ||
|
||||||
|
!strings.Contains(result.Detail, "m.third") || !strings.Contains(result.Detail, "m.fourth") {
|
||||||
|
t.Errorf("the module's result does not name the failed resource and each one left: %+v", result)
|
||||||
|
}
|
||||||
|
if other := ModuleResults(d, report, err)["o"]; other.Failed {
|
||||||
|
t.Errorf("another module was failed by it: %+v", other)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -350,6 +366,42 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A boot-time one-shot a package ships, declared stopped and enabled at boot, is satisfied while
|
||||||
|
// nothing of it runs, whatever its last run did (novox/hq issue 1126). The kernel module's
|
||||||
|
// linux-modules-cleanup.service failed on ace with "was asked to be stopped and is running": an
|
||||||
|
// inactive one-shot with ExecMainStatus=0 reads "running", and stopping it changes nothing.
|
||||||
|
func TestABootOneShotDeclaredStoppedIsSatisfiedAtRest(t *testing.T) {
|
||||||
|
for _, c := range []struct{ name, status string }{
|
||||||
|
{"exited 0: never ran, ran at boot, or unloaded", "0"},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
var asked []string
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
args = oldShape(name, args)
|
||||||
|
switch args[0] {
|
||||||
|
case "show":
|
||||||
|
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\n" +
|
||||||
|
"ExecMainStatus=" + c.status + "\n", nil
|
||||||
|
case "is-enabled":
|
||||||
|
return "enabled\n", nil
|
||||||
|
}
|
||||||
|
asked = append(asked, strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"cleanup","type":"service","unit":"linux-modules-cleanup.service","state":"stopped","boot":"enabled"}
|
||||||
|
]}`)
|
||||||
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a one-shot at rest declared stopped failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(asked) > 0 {
|
||||||
|
t.Errorf("the service manager was asked %q for a unit already as declared", asked)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
|
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
|
||||||
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
|
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
|
||||||
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
|
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
|
||||||
@@ -1225,6 +1277,52 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
|
|||||||
if reloaded < 0 || reloaded > stop {
|
if reloaded < 0 || reloaded > stop {
|
||||||
t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
|
t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A drop-in in a unit directory that no service reflects — the power module's bound on reboot.target
|
||||||
|
// (issue 1126) — is read by the service manager only after a reload: one per apply, and no restart.
|
||||||
|
// Unchanged, it reloads nothing.
|
||||||
|
units := filepath.Join(dir, "system")
|
||||||
|
saved := UnitDirs
|
||||||
|
UnitDirs = []string{units}
|
||||||
|
t.Cleanup(func() { UnitDirs = saved })
|
||||||
|
dropIn := filepath.Join(units, "reboot.target.d", "mesh-power.conf")
|
||||||
|
withDropIn := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
|
||||||
|
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]},
|
||||||
|
{"id":"units","type":"directory","path":%q,"mode":"0755"},
|
||||||
|
{"id":"drop-ins","type":"directory","path":%q,"mode":"0755"},
|
||||||
|
{"id":"bound","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"},
|
||||||
|
{"id":"bound-too","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"}
|
||||||
|
]}`, path, units, filepath.Dir(dropIn), dropIn, filepath.Join(filepath.Dir(dropIn), "other.conf")))
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
reloads int
|
||||||
|
}{{"a drop-in written", 1}, {"the same drop-in again", 0}} {
|
||||||
|
commands = nil
|
||||||
|
if _, state, err = Apply(context.Background(), archHost(t), withDropIn, state,
|
||||||
|
store.OriginCarried, run, nil, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reloads := 0
|
||||||
|
for _, c := range commands {
|
||||||
|
if strings.Contains(c, "daemon-reload") {
|
||||||
|
reloads++
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "stop thing.service") {
|
||||||
|
t.Errorf("%s: a drop-in no service reflects restarted one: %v", row.name, commands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reloads != row.reloads {
|
||||||
|
t.Errorf("%s: %d reloads, want %d; commands were %v", row.name, reloads, row.reloads, commands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A drop-in whose module is skipped behind its step, or held as found, comes back so on every apply: it moved
|
||||||
|
// nothing, and reloads nothing.
|
||||||
|
for _, action := range []string{"skipped", "held"} {
|
||||||
|
if at := unitFileMoved([]Outcome{{ID: "bound", Type: "file", Target: dropIn, Action: action}}); at != "" {
|
||||||
|
t.Errorf("a drop-in %s reloads the service manager (%s)", action, at)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -35,14 +35,22 @@ import (
|
|||||||
// enough for a desktop theme and small enough to notice.
|
// enough for a desktop theme and small enough to notice.
|
||||||
const maxArchive = 512 << 20
|
const maxArchive = 512 << 20
|
||||||
|
|
||||||
func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway) (Outcome, error) {
|
func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, held []byte, away *storeAway, journals string) (Outcome, error) {
|
||||||
out := begin(r)
|
out := begin(r)
|
||||||
out.Action = "unchanged"
|
out.Action = "unchanged"
|
||||||
|
recorded := previous.Wrote != ""
|
||||||
body, err := fetchMinding(ctx, r.Source, away)
|
previous, err := withJournal(journals, previous, r.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What the apply fetched before anything was replaced (fetchFirst), else fetched now.
|
||||||
|
body, err := held, error(nil)
|
||||||
|
if body == nil {
|
||||||
|
if body, err = fetchMinding(ctx, r.Source, away); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
sum := sha256.Sum256(body)
|
sum := sha256.Sum256(body)
|
||||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||||
if got != r.Digest {
|
if got != r.Digest {
|
||||||
@@ -78,8 +86,12 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
|
|||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
out.unpacked = &unpacked
|
out.unpacked = &unpacked
|
||||||
|
// At once, before the rest of the apply can fail or the engine stop (novox/hq issue 1585).
|
||||||
|
if err := keepJournal(journals, r.Path, got, &unpacked); err != nil {
|
||||||
|
return out, fmt.Errorf("%s was unpacked, and what it unpacked could not be written down: %w", r.Path, err)
|
||||||
|
}
|
||||||
out.Action = "updated"
|
out.Action = "updated"
|
||||||
if previous.Wrote == "" {
|
if !recorded {
|
||||||
out.Action = "created"
|
out.Action = "created"
|
||||||
}
|
}
|
||||||
out.Detail = fmt.Sprintf("%d file(s)", written)
|
out.Detail = fmt.Sprintf("%d file(s)", written)
|
||||||
@@ -290,3 +302,7 @@ func unpack(body []byte, into string) (int, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fetch is an archive at source as the host fetches every archive it unpacks, at most maxArchive bytes: what the node's
|
||||||
|
// own composer reads a moved module's templates archive with (issue 1126). Its digest is the caller's to check.
|
||||||
|
func Fetch(ctx context.Context, source string) ([]byte, error) { return fetch(ctx, source) }
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What an archive unpacked is written down the moment it is unpacked (novox/hq issue 1585).
|
||||||
|
//
|
||||||
|
// The node's state is saved at the end of a module's apply, so an engine that stopped, was replaced or lost its
|
||||||
|
// state between the unpack and that save held files on the machine it had no record of. The next apply then found
|
||||||
|
// its own files and refused them as ones "the mesh did not put there" (issue 162), and a module could not move to
|
||||||
|
// its new build until a person removed them. So, at once and atomically, the host keeps the digest it unpacked and
|
||||||
|
// the files and directories it placed; an apply that holds no record of the archive reads this instead of treating
|
||||||
|
// everything there as somebody's. The record in the state stays the authority: the journal is read only when there
|
||||||
|
// is none.
|
||||||
|
//
|
||||||
|
// **In the node-engine's own state directory, readable and writable by root alone, never beside the archive.** The
|
||||||
|
// archive's directory may be writable by the module's account (a home, a data directory): a journal there could be
|
||||||
|
// forged, and a root engine would then remove, or take as its own, any path the forger named. And an entry is read
|
||||||
|
// only when it names a path inside the archive's directory (filepath.IsLocal), and a parent only when it is above it.
|
||||||
|
|
||||||
|
// JournalDirName is the directory under the node's state directory that keeps the journals.
|
||||||
|
const JournalDirName = "unpacked"
|
||||||
|
|
||||||
|
type unpackJournal struct {
|
||||||
|
Target string `json:"target"`
|
||||||
|
Wrote string `json:"wrote"`
|
||||||
|
Unpacked *store.Unpacked `json:"unpacked"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// journalDir is where the journals of this apply are kept: under the state directory every applier on the machine
|
||||||
|
// shares (the one its windows are kept in). Empty for a caller with no state directory (a test), which keeps none.
|
||||||
|
func journalDir(w *Windows) string {
|
||||||
|
if w == nil || w.dir == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return filepath.Join(filepath.Dir(w.dir), JournalDirName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// journalPath names the journal of the archive at target: by the digest of its path, so no path names a file
|
||||||
|
// outside the journals' directory.
|
||||||
|
func journalPath(dir, target string) string {
|
||||||
|
sum := sha256.Sum256([]byte(filepath.Clean(target)))
|
||||||
|
return filepath.Join(dir, hex.EncodeToString(sum[:])+".json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepJournal writes down what an archive unpacked at path.
|
||||||
|
func keepJournal(dir, path, wrote string, u *store.Unpacked) error {
|
||||||
|
if dir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(unpackJournal{Target: filepath.Clean(path), Wrote: wrote, Unpacked: u})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeAtomicallyByPath(journalPath(dir, path), raw, 0o600)
|
||||||
|
}
|
||||||
|
|
||||||
|
// journalOf is what the journal says of path: false when there is none, and an error when there is one that cannot
|
||||||
|
// be read or names a path outside the archive's directory, which is never believed.
|
||||||
|
func journalOf(dir, path string) (unpackJournal, bool, error) {
|
||||||
|
if dir == "" {
|
||||||
|
return unpackJournal{}, false, nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(journalPath(dir, path))
|
||||||
|
if err != nil {
|
||||||
|
return unpackJournal{}, false, nil
|
||||||
|
}
|
||||||
|
var j unpackJournal
|
||||||
|
if err := json.Unmarshal(raw, &j); err != nil {
|
||||||
|
return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s is unreadable: %w", path, err)
|
||||||
|
}
|
||||||
|
if j.Wrote == "" || j.Unpacked == nil {
|
||||||
|
return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s names no digest or files", path)
|
||||||
|
}
|
||||||
|
if err := checkJournal(j, filepath.Clean(path)); err != nil {
|
||||||
|
return unpackJournal{}, false, err
|
||||||
|
}
|
||||||
|
return j, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkJournal refuses a journal of another target, an entry that is not a path inside the archive's directory, and
|
||||||
|
// a parent that is not above it.
|
||||||
|
func checkJournal(j unpackJournal, target string) error {
|
||||||
|
if j.Target != target {
|
||||||
|
return fmt.Errorf("the journal kept for %s names %q", target, j.Target)
|
||||||
|
}
|
||||||
|
for _, rel := range append(append([]string(nil), j.Unpacked.Files...), j.Unpacked.Dirs...) {
|
||||||
|
if !filepath.IsLocal(filepath.FromSlash(rel)) {
|
||||||
|
return fmt.Errorf("the journal of what was unpacked at %s names %q, which is not inside it: not believed", target, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range j.Unpacked.Parents {
|
||||||
|
if p = filepath.Clean(p); !filepath.IsAbs(p) || p == target || !strings.HasPrefix(target, p+string(filepath.Separator)) {
|
||||||
|
return fmt.Errorf("the journal of what was unpacked at %s names the parent %q, which is not above it: not believed", target, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withJournal is previous, or, when the state holds no record of the archive, what the journal says the host
|
||||||
|
// unpacked at path. A journal not believed is an error: the apply stops and says so rather than guess.
|
||||||
|
func withJournal(dir string, previous store.Applied, path string) (store.Applied, error) {
|
||||||
|
if previous.Wrote != "" {
|
||||||
|
return previous, nil
|
||||||
|
}
|
||||||
|
j, ok, err := journalOf(dir, path)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return previous, err
|
||||||
|
}
|
||||||
|
previous.Wrote, previous.Target, previous.Unpacked = j.Wrote, path, j.Unpacked
|
||||||
|
return previous, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropJournal removes the journal of the archive at target, once the archive is taken off; whether the state's
|
||||||
|
// record of it still said what it unpacked or not.
|
||||||
|
func dropJournal(dir, target string) {
|
||||||
|
if dir == "" || target == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = os.Remove(journalPath(dir, target))
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package apply
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -152,6 +153,98 @@ func TestAnArchiveDoesNotWriteOverAFileItFound(t *testing.T) {
|
|||||||
if entries, _ := os.ReadDir(dir); len(entries) != 1 {
|
if entries, _ := os.ReadDir(dir); len(entries) != 1 {
|
||||||
t.Fatalf("%d entries beside the directory", len(entries))
|
t.Fatalf("%d entries beside the directory", len(entries))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 1585: what the host unpacked is written down at once, in its own state directory, so an archive whose
|
||||||
|
// record in the state was lost is still the host's: the next version replaces its files, drops the ones it no
|
||||||
|
// longer has and leaves a file somebody added. A file the host never unpacked still refuses it, and so does a
|
||||||
|
// journal naming a path outside the archive's directory. The journal goes with the archive, record lost or not.
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
journaled bool // the first version was unpacked by the host
|
||||||
|
foreign string
|
||||||
|
forged func(j *unpackJournal)
|
||||||
|
refused string
|
||||||
|
}{
|
||||||
|
{"unpacked by the host, record lost", true, "", nil, ""},
|
||||||
|
{"unpacked by the host, somebody's file beside", true, "mine.txt", nil, ""},
|
||||||
|
{"never unpacked by the host", false, "", nil, "did not put there"},
|
||||||
|
{"a journal naming a path outside", true, "", func(j *unpackJournal) {
|
||||||
|
j.Unpacked.Files = append(j.Unpacked.Files, "../../etc/passwd")
|
||||||
|
}, "not inside it"},
|
||||||
|
{"a journal naming a parent not above", true, "", func(j *unpackJournal) {
|
||||||
|
j.Unpacked.Parents = []string{"/etc"}
|
||||||
|
}, "not above it"},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
dir, state := t.TempDir(), t.TempDir()
|
||||||
|
windows, journals := WindowsIn(state), filepath.Join(state, JournalDirName)
|
||||||
|
applyIt := func(decl string, known store.State) (store.State, error) {
|
||||||
|
_, after, err := ApplyMindingWindows(context.Background(), archHost(t), declare(t, decl), known,
|
||||||
|
store.OriginDeclared, noServices, nil, nil, nil, windows)
|
||||||
|
return after, err
|
||||||
|
}
|
||||||
|
target := filepath.Join(dir, "tools")
|
||||||
|
first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "")
|
||||||
|
if row.journaled {
|
||||||
|
if _, err := applyIt(first, store.State{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if entries, _ := os.ReadDir(dir); len(entries) != 1 {
|
||||||
|
t.Fatalf("%d entries beside the archive: the journal is the state directory's", len(entries))
|
||||||
|
}
|
||||||
|
info, err := os.Stat(journalPath(journals, target))
|
||||||
|
if err != nil || info.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("no journal of root's alone after the unpack: %v %v", info, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
os.MkdirAll(target, 0o755)
|
||||||
|
os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644)
|
||||||
|
}
|
||||||
|
if row.forged != nil {
|
||||||
|
j, _, err := journalOf(journals, target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
row.forged(&j)
|
||||||
|
raw, _ := json.Marshal(j)
|
||||||
|
os.WriteFile(journalPath(journals, target), raw, 0o600)
|
||||||
|
}
|
||||||
|
if row.foreign != "" {
|
||||||
|
os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644)
|
||||||
|
}
|
||||||
|
second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "")
|
||||||
|
// The state is empty: the record of the first unpack was lost.
|
||||||
|
after, err := applyIt(second, store.State{})
|
||||||
|
if row.refused != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
||||||
|
t.Fatalf("refused? %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" {
|
||||||
|
t.Fatalf("bin is %q", got)
|
||||||
|
}
|
||||||
|
if exists(filepath.Join(target, "old")) {
|
||||||
|
t.Fatal("a file the new version no longer has is still there")
|
||||||
|
}
|
||||||
|
if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) {
|
||||||
|
t.Fatal("somebody's file went")
|
||||||
|
}
|
||||||
|
// Taken off, with a record that no longer says what it unpacked: the journal goes too.
|
||||||
|
for i := range after.Resources {
|
||||||
|
after.Resources[i].Unpacked = nil
|
||||||
|
}
|
||||||
|
if _, err := applyIt(`{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`, after); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if exists(journalPath(journals, target)) {
|
||||||
|
t.Fatal("the journal outlived the archive")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestADirectoryTheHostMadeGoesOnlyWhenEmpty(t *testing.T) {
|
func TestADirectoryTheHostMadeGoesOnlyWhenEmpty(t *testing.T) {
|
||||||
|
|||||||
@@ -364,7 +364,16 @@ func mergeInto(fresh, path, owner string, files, dirs []string, o ours) (store.U
|
|||||||
// module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has
|
// module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has
|
||||||
// open stays readable until it closes it. The runtime is told the module went by its own membership,
|
// open stays readable until it closes it. The runtime is told the module went by its own membership,
|
||||||
// not by the files disappearing.
|
// not by the files disappearing.
|
||||||
func removeArchive(a store.Applied) (string, string, error) {
|
func removeArchive(a store.Applied, journals string) (string, string, error) {
|
||||||
|
action, detail, err := removeArchiveFiles(a)
|
||||||
|
// And its journal (issue 1585), once the record goes: whether the record said what it unpacked or not.
|
||||||
|
if err == nil {
|
||||||
|
dropJournal(journals, a.Target)
|
||||||
|
}
|
||||||
|
return action, detail, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeArchiveFiles(a store.Applied) (string, string, error) {
|
||||||
if store.IsFormer(a.ID) {
|
if store.IsFormer(a.ID) {
|
||||||
// The version before is what a rollback starts (ADR 0141) and what a reader may still have
|
// The version before is what a rollback starts (ADR 0141) and what a reader may still have
|
||||||
// open; the launcher retires the host's own versions, not the apply (issue 194).
|
// open; the launcher retires the host's own versions, not the apply (issue 194).
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every artifact an apply fetches is in hand before anything is replaced** (novox/hq issue 1966, 2026-11-08 23:25 UTC).
|
||||||
|
//
|
||||||
|
// An archive and a process are fetched from the artifact store on their own turn, and a container is replaced on its
|
||||||
|
// own: so a module whose container serves the store (distribution) had that container replaced, cut off from where it
|
||||||
|
// was reached, and then failed to fetch its program from the store it had just cut off, left half applied with the
|
||||||
|
// store unreachable from every other machine. pullFirst does this for a module's images (ENG-7); fetchFirst does it for
|
||||||
|
// its archives and processes, before any of the apply's resources is applied or any orphan removed.
|
||||||
|
//
|
||||||
|
// A fetch that fails (the store does not answer for the apply's patience, or what arrived is not what was declared)
|
||||||
|
// gates the resource's module, as a pull that fails does: nothing of the module is replaced or removed, the failure names
|
||||||
|
// the artifact, and the rest of the machine is applied. A resource of no module keeps its failure for its own turn.
|
||||||
|
|
||||||
|
// fetchedArtifact is one archive's or process's bytes, fetched and checked before the apply began, or why it was not.
|
||||||
|
type fetchedArtifact struct {
|
||||||
|
body []byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifactOf is the archive's or process's source and digest, and whether the resource is one the apply fetches.
|
||||||
|
func artifactOf(r declaration.Resource) (source, digest string, ok bool) {
|
||||||
|
switch res := r.(type) {
|
||||||
|
case *declaration.Archive:
|
||||||
|
return res.Source, res.Digest, res.Source != ""
|
||||||
|
case *declaration.Process:
|
||||||
|
return res.Source, res.Digest, res.Source != ""
|
||||||
|
}
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchFirst fetches each archive and process the declaration applies, checks each against its declared digest, and
|
||||||
|
// keeps the bytes for its turn (in.fetched). It is called before the loop that applies, after pullFirst.
|
||||||
|
func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declaration.Declaration, in *inputs,
|
||||||
|
failures *[]*Error, gated map[string]bool, gatedWhy map[string]string, log func(string)) {
|
||||||
|
in.fetched = map[string]fetchedArtifact{}
|
||||||
|
bySource := map[string]fetchedArtifact{}
|
||||||
|
for _, resource := range ordered {
|
||||||
|
source, digest, isArtifact := artifactOf(resource)
|
||||||
|
if !isArtifact {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, ours := d.ModuleOfID(resource.Identity())
|
||||||
|
if ours && gated[module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// On an adopted node, what a module not yet taken declares may be held as found and never fetched (hold.go): its
|
||||||
|
// own turn fetches it, as before.
|
||||||
|
if d.Adoption != nil {
|
||||||
|
if _, untaken := d.Adoption.UntakenModuleOf(resource.Identity()); untaken || runsIn(resource) != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What the module's done build kept on a build node is fetched from there, not from the store (design 47).
|
||||||
|
cacheKey, kept := source, false
|
||||||
|
if kf := keptFetchFrom(ctx); kf != nil && ours {
|
||||||
|
cacheKey = "kept " + module + " " + digest
|
||||||
|
if _, seen := bySource[cacheKey]; seen {
|
||||||
|
kept = true
|
||||||
|
} else if body, isKept, err := kf(ctx, module, digest); isKept {
|
||||||
|
kept = true
|
||||||
|
bySource[cacheKey] = fetchedArtifact{body: body, err: err}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, seen := bySource[cacheKey]
|
||||||
|
if !seen && !kept {
|
||||||
|
got.body, got.err = fetchMinding(ctx, source, in.away)
|
||||||
|
bySource[cacheKey] = got
|
||||||
|
}
|
||||||
|
if got.err == nil {
|
||||||
|
sum := sha256.Sum256(got.body)
|
||||||
|
if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest {
|
||||||
|
got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked",
|
||||||
|
source, digest, have)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got.err == nil {
|
||||||
|
in.fetched[resource.Identity()] = got
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !ours {
|
||||||
|
// Nothing to gate: the resource fails on its own turn with this error.
|
||||||
|
in.fetched[resource.Identity()] = got
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err := fmt.Errorf("not replacing anything of its module, which keeps running what it ran: %w", got.err)
|
||||||
|
*failures = append(*failures, &Error{Resource: resource.Identity(), Err: err})
|
||||||
|
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err))
|
||||||
|
gated[module] = true
|
||||||
|
gatedWhy[module] = "its artifact " + source + " could not be fetched, so nothing of it was replaced " +
|
||||||
|
"(novox/hq issue 1966)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifact is the bytes fetchFirst holds for a resource, taken once so they are not kept past their turn; nil and no
|
||||||
|
// error when it holds none (an apply that did not fetch first: its applier fetches itself).
|
||||||
|
func (in inputs) artifact(id string) ([]byte, error) {
|
||||||
|
got, ok := in.fetched[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
delete(in.fetched, id)
|
||||||
|
return got.body, got.err
|
||||||
|
}
|
||||||
@@ -45,6 +45,30 @@ func WriteFilled(
|
|||||||
log func(string),
|
log func(string),
|
||||||
keep Keep,
|
keep Keep,
|
||||||
windows *Windows,
|
windows *Windows,
|
||||||
|
) (report Report, _ store.State, _ error) {
|
||||||
|
return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filled is what the node-engine did to a filled file's content before its write.
|
||||||
|
type Filled struct {
|
||||||
|
// Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570
|
||||||
|
// clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is.
|
||||||
|
Secret bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled).
|
||||||
|
func WriteFilledWith(
|
||||||
|
ctx context.Context,
|
||||||
|
sys system.System,
|
||||||
|
d *declaration.Declaration,
|
||||||
|
known store.State,
|
||||||
|
path string,
|
||||||
|
content []byte,
|
||||||
|
run Runner,
|
||||||
|
log func(string),
|
||||||
|
keep Keep,
|
||||||
|
windows *Windows,
|
||||||
|
filled Filled,
|
||||||
) (report Report, _ store.State, _ error) {
|
) (report Report, _ store.State, _ error) {
|
||||||
if log == nil {
|
if log == nil {
|
||||||
log = func(string) {}
|
log = func(string) {}
|
||||||
@@ -65,13 +89,13 @@ func WriteFilled(
|
|||||||
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
|
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
|
||||||
"is written for exactly one, so nothing was written", len(declared), path)}
|
"is written for exactly one, so nothing was written", len(declared), path)}
|
||||||
}
|
}
|
||||||
file := filledAs(declared[0], content)
|
file := filledAs(declared[0], content, filled.Secret)
|
||||||
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
||||||
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
||||||
"yet taken (%s), so nothing was written", path, held.Why)}
|
"yet taken (%s), so nothing was written", path, held.Why)}
|
||||||
}
|
}
|
||||||
|
|
||||||
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
||||||
groups: groupsWanted(d.Resources), handed: Handed{}}
|
groups: groupsWanted(d.Resources), handed: Handed{}}
|
||||||
in.refused = refusedPlaces(d.Resources)
|
in.refused = refusedPlaces(d.Resources)
|
||||||
in.agentHomes = rootNeverHomes(d)
|
in.agentHomes = rootNeverHomes(d)
|
||||||
@@ -163,10 +187,10 @@ func WriteFilled(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// filledAs is the declared file with the filled content in place of the declaration's.
|
// filledAs is the declared file with the filled content in place of the declaration's.
|
||||||
func filledAs(declared *declaration.File, content []byte) *declaration.File {
|
func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File {
|
||||||
f := *declared
|
f := *declared
|
||||||
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
|
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
|
||||||
if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
|
if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) {
|
||||||
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
|
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
|
||||||
f.Mode = "0600"
|
f.Mode = "0600"
|
||||||
}
|
}
|
||||||
|
|||||||
+85
-37
@@ -283,7 +283,7 @@ func predecessor(t *testing.T) (dir, page string, m *machine) {
|
|||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
|
func caseAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
|
|
||||||
@@ -318,7 +318,7 @@ func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
|
func caseAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
|
|
||||||
@@ -338,7 +338,7 @@ func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
|
func caseWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
|
||||||
// Assigning prepares: what the module declares that is not there is made.
|
// Assigning prepares: what the module declares that is not there is made.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
page := filepath.Join(dir, "index.html")
|
page := filepath.Join(dir, "index.html")
|
||||||
@@ -355,7 +355,7 @@ func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFileThisHostWroteIsNotFound(t *testing.T) {
|
func caseAFileThisHostWroteIsNotFound(t *testing.T) {
|
||||||
// Found means present with no record. A record of any origin — carried or declared, this life
|
// Found means present with no record. A record of any origin — carried or declared, this life
|
||||||
// of the node or an earlier one — means this host wrote it.
|
// of the node or an earlier one — means this host wrote it.
|
||||||
for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
|
for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
|
||||||
@@ -372,7 +372,7 @@ func TestAFileThisHostWroteIsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAContainerAHostMadeIsNotFound(t *testing.T) {
|
func caseAContainerAHostMadeIsNotFound(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
m.containers["hello-web"].spec = "some-spec"
|
m.containers["hello-web"].spec = "some-spec"
|
||||||
report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
@@ -381,7 +381,7 @@ func TestAContainerAHostMadeIsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
|
func caseTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
|
||||||
// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
|
// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
|
||||||
// as a module by name. It is the mesh's own and must never read as a predecessor's.
|
// as a module by name. It is the mesh's own and must never read as a predecessor's.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -398,7 +398,7 @@ func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
|
func caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
h, _ := state.HeldAt("hello-web.page")
|
h, _ := state.HeldAt("hello-web.page")
|
||||||
@@ -426,7 +426,7 @@ func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
|
func caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
|
|
||||||
@@ -447,7 +447,7 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
|
func caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
first, _ := state.HeldAt("hello-web.page")
|
first, _ := state.HeldAt("hello-web.page")
|
||||||
@@ -482,7 +482,7 @@ func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
|
func caseACarriedApplyLetsNoHoldGo(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
|
||||||
carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
|
carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
|
||||||
@@ -496,7 +496,7 @@ func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
|
func caseAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
d := adopted(t, untakenWeb, webResources(page))
|
d := adopted(t, untakenWeb, webResources(page))
|
||||||
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
@@ -520,7 +520,7 @@ func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
|
func caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
|
||||||
for _, c := range []struct {
|
for _, c := range []struct {
|
||||||
change func(*machine)
|
change func(*machine)
|
||||||
want string
|
want string
|
||||||
@@ -547,7 +547,7 @@ func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
|
func caseAHeldFileThatVanishesIsNotCreated(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
d := adopted(t, untakenWeb, webResources(page))
|
d := adopted(t, untakenWeb, webResources(page))
|
||||||
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
@@ -563,7 +563,7 @@ func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
|
func caseAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
|
||||||
// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
|
// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
|
d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
|
||||||
@@ -585,7 +585,7 @@ func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
|
func caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
conf := filepath.Join(dir, "nftables.conf")
|
conf := filepath.Join(dir, "nftables.conf")
|
||||||
_ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644)
|
_ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644)
|
||||||
@@ -621,7 +621,7 @@ func untaken(ids ...string) string {
|
|||||||
return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
|
return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
|
func caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
data := filepath.Join(dir, "data")
|
data := filepath.Join(dir, "data")
|
||||||
if err := os.Mkdir(data, 0o700); err != nil {
|
if err := os.Mkdir(data, 0o700); err != nil {
|
||||||
@@ -652,7 +652,7 @@ func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
|
func caseADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
|
||||||
// A file's parent is made as the file is written; what the mesh made is not found.
|
// A file's parent is made as the file is written; what the mesh made is not found.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
data := filepath.Join(dir, "data")
|
data := filepath.Join(dir, "data")
|
||||||
@@ -671,7 +671,7 @@ func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
|
func caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
conf := filepath.Join(dir, "hello.conf")
|
conf := filepath.Join(dir, "hello.conf")
|
||||||
m := &machine{containers: map[string]*fakeContainer{},
|
m := &machine{containers: map[string]*fakeContainer{},
|
||||||
@@ -695,7 +695,7 @@ func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
|
func caseAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
|
||||||
// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
|
// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
conf := filepath.Join(dir, "daemon.json")
|
conf := filepath.Join(dir, "daemon.json")
|
||||||
@@ -716,7 +716,7 @@ func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
|
func caseAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
|
||||||
// The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit
|
// The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit
|
||||||
// the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and
|
// the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and
|
||||||
// the private network never came up. Found is what the machine runs.
|
// the private network never came up. Found is what the machine runs.
|
||||||
@@ -738,7 +738,7 @@ func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
|
func caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
|
||||||
// No unit before the apply: nothing of a predecessor's to hold.
|
// No unit before the apply: nothing of a predecessor's to hold.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
|
m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
|
||||||
@@ -750,7 +750,7 @@ func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
|
func caseAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
data := filepath.Join(dir, "predecessor-data")
|
data := filepath.Join(dir, "predecessor-data")
|
||||||
if err := os.Mkdir(data, 0o700); err != nil {
|
if err := os.Mkdir(data, 0o700); err != nil {
|
||||||
@@ -788,7 +788,7 @@ func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
|
func caseAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
data := filepath.Join(dir, "data")
|
data := filepath.Join(dir, "data")
|
||||||
m := &machine{containers: map[string]*fakeContainer{}}
|
m := &machine{containers: map[string]*fakeContainer{}}
|
||||||
@@ -801,7 +801,7 @@ func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
|
func caseARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
|
step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
|
||||||
"run-once":true,"network":"container:hello-web"}`
|
"run-once":true,"network":"container:hello-web"}`
|
||||||
@@ -818,7 +818,7 @@ func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
|
func caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
|
||||||
// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
|
// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
|
||||||
// host holds one anyway, since what it would run in is the predecessor's.
|
// host holds one anyway, since what it would run in is the predecessor's.
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
@@ -850,7 +850,7 @@ func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
|
|||||||
|
|
||||||
const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000"
|
const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000"
|
||||||
|
|
||||||
func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
|
func caseAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
at := filepath.Join(dir, "site")
|
at := filepath.Join(dir, "site")
|
||||||
if err := os.Mkdir(at, 0o750); err != nil {
|
if err := os.Mkdir(at, 0o750); err != nil {
|
||||||
@@ -874,7 +874,7 @@ func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
|
func caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
was := unitDir
|
was := unitDir
|
||||||
unitDir = dir
|
unitDir = dir
|
||||||
@@ -899,7 +899,7 @@ func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
|
func caseAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}}
|
m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}}
|
||||||
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"),
|
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"),
|
||||||
@@ -915,7 +915,7 @@ func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
|
func caseMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
|
||||||
// The runtime's socket, the kernel's filesystems and the clock are on every machine; a
|
// The runtime's socket, the kernel's filesystems and the clock are on every machine; a
|
||||||
// container mounting them shares nothing a predecessor kept (novox/hq ADR 0103).
|
// container mounting them shares nothing a predecessor kept (novox/hq ADR 0103).
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -950,7 +950,7 @@ func (u *unreachable) run(_ context.Context, name string, args ...string) (strin
|
|||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
|
func caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
u := &unreachable{}
|
u := &unreachable{}
|
||||||
d := adopted(t, untaken("hello-web.server"),
|
d := adopted(t, untaken("hello-web.server"),
|
||||||
@@ -970,7 +970,7 @@ func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
|
func caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
|
d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
|
||||||
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
@@ -988,7 +988,7 @@ func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
|
func caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
args = oldShape(name, args)
|
args = oldShape(name, args)
|
||||||
@@ -1014,7 +1014,7 @@ func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
|
func caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
|
||||||
// A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put
|
// A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put
|
||||||
// back a service somebody took down on purpose (novox/hq ADR 0103).
|
// back a service somebody took down on purpose (novox/hq ADR 0103).
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -1035,7 +1035,7 @@ func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
|
func caseAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
|
||||||
// The predecessor's own service from a package, running: not the mesh's to restart.
|
// The predecessor's own service from a package, running: not the mesh's to restart.
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
conf := filepath.Join(dir, "hello.conf")
|
conf := filepath.Join(dir, "hello.conf")
|
||||||
@@ -1054,7 +1054,7 @@ func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
|
func caseASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
|
||||||
// Held, let go when the module was unassigned, rewritten by the predecessor, found again:
|
// Held, let go when the module was unassigned, rewritten by the predecessor, found again:
|
||||||
// both originals are kept, and the report names the one it kept (novox/hq ADR 0100).
|
// both originals are kept, and the report names the one it kept (novox/hq ADR 0100).
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -1083,7 +1083,7 @@ func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
|
func caseAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
|
||||||
dir, page, m := predecessor(t)
|
dir, page, m := predecessor(t)
|
||||||
_, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)),
|
_, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)),
|
||||||
store.State{}, m, dir)
|
store.State{}, m, dir)
|
||||||
@@ -1111,7 +1111,7 @@ func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
|
func caseAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
|
||||||
// An access record says the mesh set permissions on a path it does not own — which is what it
|
// An access record says the mesh set permissions on a path it does not own — which is what it
|
||||||
// does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103).
|
// does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103).
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -1144,3 +1144,51 @@ func asideShape(args []string) []string {
|
|||||||
}
|
}
|
||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHold runs the cases of this file as subtests of one test (the test budget: one test, a table of cases).
|
||||||
|
func TestHold(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
run func(*testing.T)
|
||||||
|
}{
|
||||||
|
{"AFoundFileOfAnUntakenModuleIsKeptAsItIs", caseAFoundFileOfAnUntakenModuleIsKeptAsItIs},
|
||||||
|
{"AFoundContainerOfAnUntakenModuleIsNotReplaced", caseAFoundContainerOfAnUntakenModuleIsNotReplaced},
|
||||||
|
{"WhatIsNotFoundIsCreatedWhenAssigned", caseWhatIsNotFoundIsCreatedWhenAssigned},
|
||||||
|
{"AFileThisHostWroteIsNotFound", caseAFileThisHostWroteIsNotFound},
|
||||||
|
{"AContainerAHostMadeIsNotFound", caseAContainerAHostMadeIsNotFound},
|
||||||
|
{"TheGenesisStoreAdoptedInPlaceIsNotFound", caseTheGenesisStoreAdoptedInPlaceIsNotFound},
|
||||||
|
{"TakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives", caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives},
|
||||||
|
{"AHeldFileIsNeverRemovedWhenItsModuleIsUnassigned", caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned},
|
||||||
|
{"AHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns", caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns},
|
||||||
|
{"ACarriedApplyLetsNoHoldGo", caseACarriedApplyLetsNoHoldGo},
|
||||||
|
{"AHeldFileRewrittenIsReportedAndNotReverted", caseAHeldFileRewrittenIsReportedAndNotReverted},
|
||||||
|
{"AHeldContainerStoppedOrReplacedIsReportedAndNotRestarted", caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted},
|
||||||
|
{"AHeldFileThatVanishesIsNotCreated", caseAHeldFileThatVanishesIsNotCreated},
|
||||||
|
{"AConvergedNodeStillReplacesWhatItFinds", caseAConvergedNodeStillReplacesWhatItFinds},
|
||||||
|
{"AFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt", caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt},
|
||||||
|
{"AFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents", caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents},
|
||||||
|
{"ADirectoryMadeInTheSameApplyIsNotFound", caseADirectoryMadeInTheSameApplyIsNotFound},
|
||||||
|
{"AFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted", caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted},
|
||||||
|
{"AHeldServiceIsStillReloadedButNeverRestarted", caseAHeldServiceIsStillReloadedButNeverRestarted},
|
||||||
|
{"AUnitAPackageOnlyShipsIsNotFound", caseAUnitAPackageOnlyShipsIsNotFound},
|
||||||
|
{"AServiceWhoseUnitIsNotThereIsAppliedAsUsual", caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual},
|
||||||
|
{"AContainerThatWouldMountFoundDataIsNotCreated", caseAContainerThatWouldMountFoundDataIsNotCreated},
|
||||||
|
{"AContainerMountingWhatTheMeshMadeIsCreated", caseAContainerMountingWhatTheMeshMadeIsCreated},
|
||||||
|
{"ARunOnceStepInAHeldContainerIsHeld", caseARunOnceStepInAHeldContainerIsHeld},
|
||||||
|
{"AnActionInAHeldContainerIsHeldUntilItsModuleIsTaken", caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken},
|
||||||
|
{"AnArchiveOverSomethingFoundIsNotUnpacked", caseAnArchiveOverSomethingFoundIsNotUnpacked},
|
||||||
|
{"AProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted", caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted},
|
||||||
|
{"AUserFoundOnTheMachineKeepsItsShellAndGroups", caseAUserFoundOnTheMachineKeepsItsShellAndGroups},
|
||||||
|
{"MountingTheMachinesOwnPlumbingIsNotFoundData", caseMountingTheMachinesOwnPlumbingIsNotFoundData},
|
||||||
|
{"ARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer", caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer},
|
||||||
|
{"AHeldContainerStaysHeldWhenTheRuntimeCannotAnswer", caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer},
|
||||||
|
{"AVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer", caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer},
|
||||||
|
{"AUnitSomebodyInstalledIsHeldWhateverStateItIsIn", caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn},
|
||||||
|
{"APackagedUnitTheMachineUsesIsStillHeld", caseAPackagedUnitTheMachineUsesIsStillHeld},
|
||||||
|
{"ASecondOriginalAtTheSamePathIsKeptToo", caseASecondOriginalAtTheSamePathIsKeptToo},
|
||||||
|
{"AHoldLetGoAndFoundAgainKeepsBothOriginals", caseAHoldLetGoAndFoundAgainKeepsBothOriginals},
|
||||||
|
{"APathTheMeshOnlySetAccessOnIsStillFound", caseAPathTheMeshOnlySetAccessOnIsStillFound},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, c.run)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ func TestRemovalThroughALinkUnderAHomeIsRefused(t *testing.T) {
|
|||||||
home, outside := aLinkedHome(t)
|
home, outside := aLinkedHome(t)
|
||||||
link(t, outside, filepath.Join(home, ".claude"))
|
link(t, outside, filepath.Join(home, ".claude"))
|
||||||
_, _, err := remove(context.Background(), archHost(t), store.Applied{ID: "m.f", Type: "file",
|
_, _, err := remove(context.Background(), archHost(t), store.Applied{ID: "m.f", Type: "file",
|
||||||
Target: filepath.Join(home, ".claude", "shadow")}, nil, nil)
|
Target: filepath.Join(home, ".claude", "shadow")}, nil, nil, "")
|
||||||
mustBeLinkRefusal(t, err)
|
mustBeLinkRefusal(t, err)
|
||||||
if _, err := os.Stat(filepath.Join(outside, "shadow")); err != nil {
|
if _, err := os.Stat(filepath.Join(outside, "shadow")); err != nil {
|
||||||
t.Fatal("removed through the link")
|
t.Fatal("removed through the link")
|
||||||
|
|||||||
@@ -112,18 +112,77 @@ func TestTheRegistryMovesToTheRuntimesModuleWithoutLeavingTheList(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// And the preview says the same before it happens.
|
// A unit two records name, under opposite conditions, passes between them (design 42 zfs-owns-zfs §2, answer A-21): the
|
||||||
func TestThePlanForgetsARecordOfAUnitStillHeld(t *testing.T) {
|
// going record is forgotten, the preview says so before it happens, and what it found of the unit before the mesh is what
|
||||||
path := filepath.Join(t.TempDir(), "daemon.json")
|
// the staying record gives back when its module leaves, not what the mesh made of it.
|
||||||
var commands []string
|
func TestARecordOfAUnitStillHeldIsForgottenAndPassesWhatItFound(t *testing.T) {
|
||||||
_, state, err := Apply(context.Background(), archHost(t), runtimeDecl(t, path, true), store.State{},
|
t.Run("the preview forgets it", func(t *testing.T) {
|
||||||
store.OriginDeclared, recordingServices(&commands), nil, nil)
|
path := filepath.Join(t.TempDir(), "daemon.json")
|
||||||
if err != nil {
|
var commands []string
|
||||||
t.Fatal(err)
|
_, state, err := Apply(context.Background(), archHost(t), runtimeDecl(t, path, true), store.State{},
|
||||||
}
|
store.OriginDeclared, recordingServices(&commands), nil, nil)
|
||||||
for _, step := range Plan(runtimeDecl(t, path, false), state, store.OriginDeclared) {
|
if err != nil {
|
||||||
if step.ID == "networking.registry-trust-reload" && step.Verb != "forget" {
|
t.Fatal(err)
|
||||||
t.Errorf("the plan would %s the network's record of a unit docker.runtime holds: %s", step.Verb, step.Why)
|
|
||||||
}
|
}
|
||||||
}
|
for _, step := range Plan(runtimeDecl(t, path, false), state, store.OriginDeclared) {
|
||||||
|
if step.ID == "networking.registry-trust-reload" && step.Verb != "forget" {
|
||||||
|
t.Errorf("the plan would %s the network's record of a unit docker.runtime holds: %s", step.Verb, step.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("what it found passes to the record that stays", func(t *testing.T) {
|
||||||
|
// A unit found stopped and disabled, so the mesh's first record of it started it.
|
||||||
|
active, enabled := false, false
|
||||||
|
var commands []string
|
||||||
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
args = oldShape(name, args)
|
||||||
|
line := name + " " + strings.Join(args, " ")
|
||||||
|
commands = append(commands, line)
|
||||||
|
switch {
|
||||||
|
case strings.Contains(line, " start "):
|
||||||
|
active = true
|
||||||
|
case strings.Contains(line, " stop "):
|
||||||
|
active = false
|
||||||
|
case strings.Contains(line, " enable "):
|
||||||
|
enabled = true
|
||||||
|
case strings.Contains(line, " disable "):
|
||||||
|
enabled = false
|
||||||
|
case strings.Contains(line, "is-enabled"):
|
||||||
|
if enabled {
|
||||||
|
return "enabled", nil
|
||||||
|
}
|
||||||
|
return "disabled", nil
|
||||||
|
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
||||||
|
if active {
|
||||||
|
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
||||||
|
}
|
||||||
|
return "LoadState=loaded\nActiveState=inactive\nSubState=dead", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
unit := func(id, state, boot string) *declaration.Declaration {
|
||||||
|
return parse(t, `{"declaration":1,"resources":[{"id":"`+id+`","type":"service","unit":"z.service","state":"`+
|
||||||
|
state+`","boot":"`+boot+`"}]}`)
|
||||||
|
}
|
||||||
|
apply := func(d *declaration.Declaration, known store.State) store.State {
|
||||||
|
t.Helper()
|
||||||
|
_, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return known
|
||||||
|
}
|
||||||
|
known := apply(unit("z.on", "running", "enabled"), store.State{})
|
||||||
|
known = apply(unit("z.off", "stopped", "disabled"), known) // the value changed: one record goes, the other stays
|
||||||
|
if rec, _ := known.Find("z.off"); rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
|
||||||
|
t.Fatalf("the staying record did not receive what the going one found: %+v", rec.Found)
|
||||||
|
}
|
||||||
|
commands = nil
|
||||||
|
apply(somethingElse(t), known) // the module leaves
|
||||||
|
for _, c := range commands {
|
||||||
|
if strings.Contains(c, " start ") || strings.Contains(c, " enable ") {
|
||||||
|
t.Errorf("the unit was given back as the mesh made it, not as it was found: %q", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// The build the artifact store's holder keeps on a build node (novox/hq design 47, *The artifact store's holder takes its
|
||||||
|
// own new build even when its seat is the one broken*): the holder's program cannot be fetched from the store it serves, so
|
||||||
|
// its done build names the node that keeps it and the node-engine fetches it from there, through the build seat's verb.
|
||||||
|
// How is the caller's (a hook on the apply's context, as the files' link is); what apply does is use the bytes in place of
|
||||||
|
// the source's fetch, check them against the declared digest as every fetch is, and gate the module when they cannot be had.
|
||||||
|
|
||||||
|
// KeptFetch gives the bytes of the artifact a module's done build kept on a build node, by the artifact's digest
|
||||||
|
// ("sha256:" and 64 hex digits, as a declaration names it). kept is false when the module's build kept no such artifact:
|
||||||
|
// the artifact is then fetched from its source, as before. An error says the artifact is kept and could not be had.
|
||||||
|
type KeptFetch func(ctx context.Context, module, digest string) (body []byte, kept bool, err error)
|
||||||
|
|
||||||
|
type keptFetchKey struct{}
|
||||||
|
|
||||||
|
// WithKept is ctx carrying how a kept artifact is fetched.
|
||||||
|
func WithKept(ctx context.Context, f KeptFetch) context.Context {
|
||||||
|
if f == nil {
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, keptFetchKey{}, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func keptFetchFrom(ctx context.Context) KeptFetch {
|
||||||
|
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
|
||||||
|
return f
|
||||||
|
}
|
||||||
@@ -41,6 +41,7 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str
|
|||||||
outcomes = append(append([]Outcome(nil), outcomes...), failed.Done.Outcomes...)
|
outcomes = append(append([]Outcome(nil), outcomes...), failed.Done.Outcomes...)
|
||||||
}
|
}
|
||||||
skipped := map[string]string{}
|
skipped := map[string]string{}
|
||||||
|
left := map[string][]string{}
|
||||||
for _, o := range outcomes {
|
for _, o := range outcomes {
|
||||||
module, ok := d.ModuleOfID(o.ID)
|
module, ok := d.ModuleOfID(o.ID)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -48,6 +49,7 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str
|
|||||||
}
|
}
|
||||||
if o.Action == "skipped" {
|
if o.Action == "skipped" {
|
||||||
skipped[module] = o.Detail
|
skipped[module] = o.Detail
|
||||||
|
left[module] = append(left[module], o.ID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
attempted[o.ID] = true
|
attempted[o.ID] = true
|
||||||
@@ -92,6 +94,11 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for module, why := range reasons {
|
for module, why := range reasons {
|
||||||
|
// Each resource left unapplied is named (design 42 zfs-owns-zfs §1).
|
||||||
|
if ids := left[module]; len(ids) > 0 {
|
||||||
|
sort.Strings(ids)
|
||||||
|
why = append(why, "left unapplied: "+strings.Join(ids, ", "))
|
||||||
|
}
|
||||||
sort.Strings(why)
|
sort.Strings(why)
|
||||||
out[module] = ModuleResult{Failed: true, Detail: strings.Join(why, "; ")}
|
out[module] = ModuleResult{Failed: true, Detail: strings.Join(why, "; ")}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package apply
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
@@ -116,32 +117,107 @@ func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) {
|
|||||||
// leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32:
|
// leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32:
|
||||||
// shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server,
|
// shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server,
|
||||||
// because the node's own apply removed every record no longer in it, the others holding the very same resources or not).
|
// because the node's own apply removed every record no longer in it, the others holding the very same resources or not).
|
||||||
|
//
|
||||||
|
// And a resource the node's own composition carried for a module a want still places here (a folded module of the old
|
||||||
|
// form, held) is never removed by the node's own apply when a composition leaves it out: the apply is refused whole and
|
||||||
|
// says why (issue 774, live on 2026-10-29 at 07:54 UTC on the bus node: the node's own composition, composed without what
|
||||||
|
// it ran once the kept declaration was read no more, removed the bus server, the proxy and three held modules). Only its
|
||||||
|
// want's removal lets it go.
|
||||||
func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) {
|
func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) {
|
||||||
dir := t.TempDir()
|
for _, tc := range []struct {
|
||||||
fact := filepath.Join(dir, "suffix")
|
name string
|
||||||
gone := filepath.Join(dir, "gone.conf")
|
id string // the resource the node's own composition carried, and no longer does
|
||||||
own := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n"},
|
moved bool // its module's own composition now declares it (Others)
|
||||||
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
|
// dropped: its module has a composition of its own here that no longer declares it (Others holds another of its
|
||||||
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
|
// resources); the node's own apply leaves it, and the module's own apply removes it.
|
||||||
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
|
dropped bool
|
||||||
if err != nil {
|
placed bool // a want still places its module here
|
||||||
t.Fatal(err)
|
refused bool // the node's own apply is refused whole
|
||||||
}
|
stays bool // the resource is still on the machine and recorded
|
||||||
// systemd-resolved now has a composition of its own, not applied yet; old.gone is no module's any more.
|
// wants names the modules whose want says removed (Scope.RemovedByWant); nil leaves the guard to Placed alone.
|
||||||
moved := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n","module":"systemd-resolved"}`)
|
wants []string
|
||||||
report, state, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{},
|
}{
|
||||||
Scope{Module: store.NodeModule, Others: moved.Resources}, state, store.OriginDeclared, noServices, nil, nil, nil, nil)
|
{name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true},
|
||||||
if err != nil {
|
{name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true},
|
||||||
t.Fatal(err)
|
{name: "its module's want removed", id: "nats.server-conf"},
|
||||||
}
|
// Issue 774, review of the re-land, round 1: a module folded in the node's own has no want, so Placed does not name
|
||||||
if !exists(fact) {
|
// it; in the node's own apply only a module a want removed loses its resources.
|
||||||
t.Fatalf("the moved resource was removed: %+v", outcomeOf(report, "systemd-resolved.fact-suffix"))
|
{name: "a folded module with no want", id: "nats.server-conf", wants: []string{}, refused: true, stays: true},
|
||||||
}
|
{name: "a folded module a want removed", id: "nats.server-conf", wants: []string{"nats", "old"}},
|
||||||
if exists(gone) {
|
// 2026-10-30 19:42 on the bus node: restic's manifest replaced its process measure by awake. Its record was the
|
||||||
t.Fatal("a resource no composition holds any more was kept")
|
// node's, carried when restic was folded, so restic's own apply never saw it and the node's own was refused
|
||||||
}
|
// whole every two minutes, applying nothing of the node's own.
|
||||||
if _, still := state.Find("systemd-resolved.fact-suffix"); !still {
|
{name: "its module's own composition dropped it", id: "restic.measure", dropped: true, placed: true,
|
||||||
t.Fatal("the moved resource's record was dropped, so its module's apply would not know it")
|
wants: []string{"old"}, stays: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
fact := filepath.Join(dir, "carried")
|
||||||
|
gone := filepath.Join(dir, "gone.conf")
|
||||||
|
own := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n"},
|
||||||
|
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
|
||||||
|
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
|
||||||
|
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
module, _ := declaration.ModuleGuess(tc.id)
|
||||||
|
scope := Scope{Module: store.NodeModule}
|
||||||
|
if tc.moved {
|
||||||
|
moved := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n","module":"`+module+`"}`)
|
||||||
|
scope.Others = moved.Resources
|
||||||
|
}
|
||||||
|
if tc.dropped {
|
||||||
|
later := declare(t, `{"id":"`+module+`.awake","type":"file","path":"`+filepath.Join(dir, "awake")+
|
||||||
|
`","content":"a\n","module":"`+module+`"}`)
|
||||||
|
scope.Others = later.Resources
|
||||||
|
}
|
||||||
|
if tc.placed {
|
||||||
|
scope.Placed = map[string]bool{module: true}
|
||||||
|
}
|
||||||
|
if tc.wants != nil {
|
||||||
|
scope.RemovedByWant = map[string]bool{}
|
||||||
|
for _, m := range tc.wants {
|
||||||
|
scope.RemovedByWant[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state,
|
||||||
|
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
|
if tc.refused != (err != nil) {
|
||||||
|
t.Fatalf("refused %v, want %v: %v", err != nil, tc.refused, err)
|
||||||
|
}
|
||||||
|
if tc.refused {
|
||||||
|
if !strings.Contains(err.Error(), tc.id+" of "+module) {
|
||||||
|
t.Fatalf("the refusal does not name what it kept: %v", err)
|
||||||
|
}
|
||||||
|
after = state
|
||||||
|
}
|
||||||
|
if exists(fact) != tc.stays {
|
||||||
|
t.Fatalf("the carried resource exists %v, want %v: %+v", exists(fact), tc.stays, outcomeOf(report, tc.id))
|
||||||
|
}
|
||||||
|
if _, still := after.Find(tc.id); still != tc.stays {
|
||||||
|
t.Fatalf("the carried resource's record kept %v, want %v", still, tc.stays)
|
||||||
|
}
|
||||||
|
if exists(gone) == !tc.refused {
|
||||||
|
t.Fatalf("a resource no composition holds any more exists %v, with the apply refused %v", exists(gone), tc.refused)
|
||||||
|
}
|
||||||
|
if tc.dropped {
|
||||||
|
if o := outcomeOf(report, tc.id); o.Action != "kept" {
|
||||||
|
t.Fatalf("the node's own apply did not say it left it: %+v", o)
|
||||||
|
}
|
||||||
|
later := declare(t, `{"id":"`+module+`.awake","type":"file","path":"`+filepath.Join(dir, "awake")+
|
||||||
|
`","content":"a\n","module":"`+module+`"}`)
|
||||||
|
_, last, err := ApplyModule(context.Background(), archHost(t), later, Scope{Module: module}, after,
|
||||||
|
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, still := last.Find(tc.id); still || exists(fact) {
|
||||||
|
t.Fatalf("the module's own apply left what it no longer declares: recorded %v, on the machine %v",
|
||||||
|
still, exists(fact))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
|
||||||
|
// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
|
||||||
|
// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
|
||||||
|
// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
|
||||||
|
// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
|
||||||
|
//
|
||||||
|
// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
|
||||||
|
// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
|
||||||
|
type ObjectKeep struct {
|
||||||
|
Dir string
|
||||||
|
// Now is the clock; nil is time.Now.
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptObjectsFor is how long an object of the keep is held after it was last used.
|
||||||
|
const KeptObjectsFor = 14 * 24 * time.Hour
|
||||||
|
|
||||||
|
// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
|
||||||
|
func KeepObjectsIn(dir string) *ObjectKeep {
|
||||||
|
if dir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &ObjectKeep{Dir: dir}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *ObjectKeep) now() time.Time {
|
||||||
|
if k.Now != nil {
|
||||||
|
return k.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get is the kept content of the object name when it is what f names, and marks it used.
|
||||||
|
func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
|
||||||
|
if k == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
path := filepath.Join(k.Dir, name)
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
now := k.now()
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
return body, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
|
||||||
|
func (k *ObjectKeep) put(name string, body []byte) {
|
||||||
|
if k == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := filepath.Join(k.Dir, name)
|
||||||
|
now := k.now()
|
||||||
|
if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(k.Dir, 0o700); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(k.Dir, ".put-*")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, werr := tmp.Write(body)
|
||||||
|
cerr := tmp.Close()
|
||||||
|
if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
|
||||||
|
_ = os.Remove(tmp.Name())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
k.prune(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// prune lets go of every object unused for KeptObjectsFor.
|
||||||
|
func (k *ObjectKeep) prune(now time.Time) {
|
||||||
|
entries, err := os.ReadDir(k.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
info, err := e.Info()
|
||||||
|
if err != nil || !info.Mode().IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if now.Sub(info.ModTime()) >= KeptObjectsFor {
|
||||||
|
_ = os.Remove(filepath.Join(k.Dir, e.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
|
||||||
|
// keep did not hold the object either.
|
||||||
|
func (k *ObjectKeep) never() string {
|
||||||
|
if k == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
|
||||||
|
}
|
||||||
@@ -54,6 +54,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
@@ -303,8 +304,28 @@ func splitQuoted(s string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// engineModule is the module whose resources may place in the engine's own trees.
|
// engineModuleOf answers the module whose resources may place in the engine's own trees: the module whose entry of the
|
||||||
const engineModule = "mesh-host"
|
// catalog half claims the seat node-engine (design 47 G §527), asked of the node's composer; none until one is given.
|
||||||
|
var engineModuleOf atomic.Pointer[func() string]
|
||||||
|
|
||||||
|
// UseEngineModule gives the apply guard the function that answers the node-engine's own module; nil takes it away. The
|
||||||
|
// guard fixes no name (§364, §527).
|
||||||
|
func UseEngineModule(f func() string) {
|
||||||
|
if f == nil {
|
||||||
|
engineModuleOf.Store(nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
engineModuleOf.Store(&f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isEngineModule is whether module is the node-engine's own.
|
||||||
|
func isEngineModule(module string) bool {
|
||||||
|
f := engineModuleOf.Load()
|
||||||
|
if f == nil || module == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return (*f)() == module
|
||||||
|
}
|
||||||
|
|
||||||
// passwdFile is the user database homes are read from. A variable so a test names its own.
|
// passwdFile is the user database homes are read from. A variable so a test names its own.
|
||||||
var passwdFile = "/etc/passwd"
|
var passwdFile = "/etc/passwd"
|
||||||
@@ -365,11 +386,15 @@ func below(path, dir string) bool {
|
|||||||
// atOrBelow says whether path is dir or below it.
|
// atOrBelow says whether path is dir or below it.
|
||||||
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
|
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
|
||||||
|
|
||||||
|
// evalSymlinks follows the links of an existing path; a variable so a test judges paths whose machine it does not touch (a mount
|
||||||
|
// of the machine it runs on may not answer).
|
||||||
|
var evalSymlinks = filepath.EvalSymlinks
|
||||||
|
|
||||||
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
|
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
|
||||||
func resolved(path string) string {
|
func resolved(path string) string {
|
||||||
rest := ""
|
rest := ""
|
||||||
for p := path; ; p = filepath.Dir(p) {
|
for p := path; ; p = filepath.Dir(p) {
|
||||||
if real, err := filepath.EvalSymlinks(p); err == nil {
|
if real, err := evalSymlinks(p); err == nil {
|
||||||
return filepath.Clean(filepath.Join(real, rest))
|
return filepath.Clean(filepath.Join(real, rest))
|
||||||
}
|
}
|
||||||
if filepath.Dir(p) == p {
|
if filepath.Dir(p) == p {
|
||||||
@@ -531,7 +556,7 @@ func refuseOne(path string, kind placing, module, owner string, declared map[str
|
|||||||
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
|
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if module != engineModule {
|
if !isEngineModule(module) {
|
||||||
for _, tree := range engineTrees {
|
for _, tree := range engineTrees {
|
||||||
if atOrBelow(path, tree) {
|
if atOrBelow(path, tree) {
|
||||||
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
|
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
|
||||||
|
|||||||
@@ -178,6 +178,11 @@ func TestAFoundDirectoryIsUsedAsFound(t *testing.T) {
|
|||||||
// The module's own directories keep being placed: below /var/lib, below /etc as root's, a container's mount of
|
// The module's own directories keep being placed: below /var/lib, below /etc as root's, a container's mount of
|
||||||
// the machine's docker socket.
|
// the machine's docker socket.
|
||||||
func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) {
|
func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) {
|
||||||
|
// The paths below are judged as written: no link of them is followed on the machine the test runs on, whose mounts of them
|
||||||
|
// (a share that does not answer) it does not depend on.
|
||||||
|
was := evalSymlinks
|
||||||
|
evalSymlinks = func(p string) (string, error) { return p, nil }
|
||||||
|
t.Cleanup(func() { evalSymlinks = was })
|
||||||
for _, path := range []string{"/var/lib/notes/data", "/etc/systemd/logind.conf.d", "/var/run/fail2ban",
|
for _, path := range []string{"/var/lib/notes/data", "/etc/systemd/logind.conf.d", "/var/run/fail2ban",
|
||||||
"/srv/notes", "/mnt/plex/config", "/storage/media", "/var/lib/mesh/registry"} {
|
"/srv/notes", "/mnt/plex/config", "/storage/media", "/var/lib/mesh/registry"} {
|
||||||
if err := refusePath(path, placingDirectory, "notes", ""); err != nil {
|
if err := refusePath(path, placingDirectory, "notes", ""); err != nil {
|
||||||
@@ -193,4 +198,24 @@ func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) {
|
|||||||
if err := refusePath("/var/lib/notes/data", placingDirectory, "notes", "1001:2000"); err != nil {
|
if err := refusePath("/var/lib/notes/data", placingDirectory, "notes", "1001:2000"); err != nil {
|
||||||
t.Errorf("an owner below /var/lib: %v", err)
|
t.Errorf("an owner below /var/lib: %v", err)
|
||||||
}
|
}
|
||||||
|
// Design 47 G §364, §527: the node-engine's own trees are placed in by the module the composer says claims its seat,
|
||||||
|
// whatever that module is called, and by no other; with none said, by nobody.
|
||||||
|
tree := engineTrees[0]
|
||||||
|
t.Cleanup(func() { UseEngineModule(nil) })
|
||||||
|
for _, c := range []struct {
|
||||||
|
engine func() string
|
||||||
|
module string
|
||||||
|
placed bool
|
||||||
|
}{
|
||||||
|
{func() string { return "the-engine" }, "the-engine", true},
|
||||||
|
{func() string { return "the-engine" }, "notes", false},
|
||||||
|
{func() string { return "the-engine" }, "", false},
|
||||||
|
{func() string { return "" }, "", false},
|
||||||
|
{nil, "the-engine", false},
|
||||||
|
} {
|
||||||
|
UseEngineModule(c.engine)
|
||||||
|
if err := refusePath(tree, placingDirectory, c.module, ""); (err == nil) != c.placed {
|
||||||
|
t.Errorf("%s for module %q: %v, want placed %v", tree, c.module, err, c.placed)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+202
-6
@@ -4,9 +4,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -39,13 +43,16 @@ var daemonRoot = "/var/lib/mesh/daemons"
|
|||||||
var unitDir = "/etc/systemd/system"
|
var unitDir = "/etc/systemd/system"
|
||||||
|
|
||||||
func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||||
changed map[string]bool, previous store.Applied, away *storeAway) (Outcome, error) {
|
changed map[string]bool, previous store.Applied, held []byte, away *storeAway) (Outcome, error) {
|
||||||
out := begin(r)
|
out := begin(r)
|
||||||
out.Action = "unchanged"
|
out.Action = "unchanged"
|
||||||
|
|
||||||
body, err := fetchMinding(ctx, r.Source, away)
|
// What the apply fetched before anything was replaced (fetchFirst), else fetched now.
|
||||||
if err != nil {
|
body, err := held, error(nil)
|
||||||
return out, err
|
if body == nil {
|
||||||
|
if body, err = fetchMinding(ctx, r.Source, away); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
sum := sha256.Sum256(body)
|
sum := sha256.Sum256(body)
|
||||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||||
@@ -94,6 +101,14 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
|||||||
if r.Schedule != "" {
|
if r.Schedule != "" {
|
||||||
unit = r.Name + ".timer"
|
unit = r.Name + ".timer"
|
||||||
}
|
}
|
||||||
|
// The yielding slice is checked even then: a slice file removed by hand would leave the build unbound.
|
||||||
|
if wrote, err := placeYieldSlice(r); err != nil {
|
||||||
|
return out, err
|
||||||
|
} else if wrote {
|
||||||
|
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
|
if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
|
||||||
_ = active
|
_ = active
|
||||||
return out, nil
|
return out, nil
|
||||||
@@ -121,7 +136,12 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
|||||||
}
|
}
|
||||||
// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
|
// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
|
||||||
// previous version would be loaded by a runtime that walks a directory.
|
// previous version would be loaded by a runtime that walks a directory.
|
||||||
if err := os.RemoveAll(at); err != nil {
|
// **Retired, never deleted under its process** (novox/hq issue 734): the old process runs until
|
||||||
|
// the restart below, from this directory and in it. Deleted here, every command it started
|
||||||
|
// before the restart failed with "getwd: no such file or directory" — a builder's toolchain probe
|
||||||
|
// refused three checks on 2026-10-26. Renamed aside, its process keeps a directory that exists,
|
||||||
|
// and the build is deleted once nothing runs from it, as a judged build's is.
|
||||||
|
if err := witness.Retire(daemonRoot, r.Name, at, time.Now()); err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -189,6 +209,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
|||||||
if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
|
if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
if _, err := placeYieldSlice(r); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
@@ -240,6 +263,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
|||||||
}
|
}
|
||||||
if because != "" {
|
if because != "" {
|
||||||
out.Detail += ", restarted because " + because + " changed"
|
out.Detail += ", restarted because " + because + " changed"
|
||||||
|
} else if what := unitDifference(previous.Wrote, want); what != "" {
|
||||||
|
// Said, so a restart nobody asked for names its cause (issue 1966): the file and the input of it, never a value.
|
||||||
|
out.Detail += ", restarted because " + what + " changed"
|
||||||
}
|
}
|
||||||
if placing.Detail != "" {
|
if placing.Detail != "" {
|
||||||
out.Detail += "; " + placing.Detail
|
out.Detail += "; " + placing.Detail
|
||||||
@@ -248,6 +274,25 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NodeEnv is the variable every process the node-engine runs reads its node's name from (design 47 G A-342), the one
|
||||||
|
// mesh-sdk reads (MESH_NODE).
|
||||||
|
const NodeEnv = "MESH_NODE"
|
||||||
|
|
||||||
|
// withNode is the process as it runs on node: its own env with NodeEnv beside it (design 47 G A-342). The validator
|
||||||
|
// refuses a resource that sets it, so the mesh's value is the only one; with no node named the process is as declared.
|
||||||
|
func withNode(r *declaration.Process, node string) *declaration.Process {
|
||||||
|
if node == "" {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
p := *r
|
||||||
|
p.Env = make(map[string]string, len(r.Env)+1)
|
||||||
|
for k, v := range r.Env {
|
||||||
|
p.Env[k] = v
|
||||||
|
}
|
||||||
|
p.Env[NodeEnv] = node
|
||||||
|
return &p
|
||||||
|
}
|
||||||
|
|
||||||
// unitFor is the unit the mesh writes for a daemon.
|
// unitFor is the unit the mesh writes for a daemon.
|
||||||
//
|
//
|
||||||
// **Generated whole and never edited in place**, the same rule as every other managed file: an
|
// **Generated whole and never edited in place**, the same rule as every other managed file: an
|
||||||
@@ -277,6 +322,7 @@ func unitFor(r *declaration.Process) string {
|
|||||||
if r.User != "" {
|
if r.User != "" {
|
||||||
fmt.Fprintf(&b, "User=%s\n", r.User)
|
fmt.Fprintf(&b, "User=%s\n", r.User)
|
||||||
}
|
}
|
||||||
|
writeYields(&b, r.Yields)
|
||||||
fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
|
fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
|
||||||
if r.Schedule != "" || r.RunOnce {
|
if r.Schedule != "" || r.RunOnce {
|
||||||
// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
|
// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
|
||||||
@@ -293,6 +339,78 @@ func unitFor(r *declaration.Process) string {
|
|||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// machineCPUs is how many processors this machine has, for a share of them in a quota. A variable only so a test can
|
||||||
|
// fix it.
|
||||||
|
var machineCPUs = runtime.NumCPU
|
||||||
|
|
||||||
|
// writeYields writes how a process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): a
|
||||||
|
// build seat's holder, whose builds must never slow the operator's desktop.
|
||||||
|
//
|
||||||
|
// **Into the node's one yielding slice**, where the CPU weight and the quota are (yieldSlice): every yielding process
|
||||||
|
// of the machine shares them, so three build modules take one share together, never one each. **Per process**, what a
|
||||||
|
// slice cannot say: Nice=19 for every process it starts, and the lowest best-effort IO level. The IO level binds only
|
||||||
|
// where the disk's scheduler honours it (bfq); under `none`, the usual one for NVMe, it does nothing. Not the idle IO
|
||||||
|
// class: a disk that is never idle would starve a build for good.
|
||||||
|
//
|
||||||
|
// The share is written into the unit as a comment, so a changed share changes the unit, the daemon's identity, and the
|
||||||
|
// slice is written again.
|
||||||
|
func writeYields(b *strings.Builder, y *declaration.Yields) {
|
||||||
|
if y == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(b, "# Yields: %s, %d%% of the machine shared in %s\n", y.Priority, y.CPUShare, declaration.YieldSlice)
|
||||||
|
fmt.Fprintf(b, "Slice=%s\n", declaration.YieldSlice)
|
||||||
|
if y.Priority == declaration.YieldIdle {
|
||||||
|
b.WriteString("Nice=19\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// yieldSlice is the node's one yielding slice (declaration.YieldSlice) for a process that yields: idle CPU weight and
|
||||||
|
// IO weight 1 for an idle priority, and CPUQuota for its share of the processors this machine has (systemd counts a
|
||||||
|
// quota in percent of one processor, so half of 24 is 1200%), at least one processor. At the top of the hierarchy, so
|
||||||
|
// its weight and quota bind against the operator's user slice and the system's.
|
||||||
|
//
|
||||||
|
// The IO weight binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under `none` it does
|
||||||
|
// nothing, and the CPU bound is what holds.
|
||||||
|
//
|
||||||
|
// Every yielding process of a machine carries the node's one share (the composer's, ADR 0553 §3), so whichever writes
|
||||||
|
// it writes the same file. Left in place when nothing yields any more: an empty slice binds nothing.
|
||||||
|
func yieldSlice(y *declaration.Yields) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a yielding\n")
|
||||||
|
b.WriteString("# process is declared again (novox/hq ADR 0553).\n")
|
||||||
|
b.WriteString("[Unit]\nDescription=The mesh's yielding processes: builds, at idle priority under one share\n\n")
|
||||||
|
b.WriteString("[Slice]\n")
|
||||||
|
if y.Priority == declaration.YieldIdle {
|
||||||
|
b.WriteString("CPUWeight=idle\nIOWeight=1\n")
|
||||||
|
}
|
||||||
|
if y.CPUShare > 0 {
|
||||||
|
quota := y.CPUShare * machineCPUs()
|
||||||
|
if quota < 100 {
|
||||||
|
quota = 100
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "CPUQuota=%d%%\n", quota)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// placeYieldSlice writes the yielding slice for a process that yields, when what is there differs, and says whether it
|
||||||
|
// wrote: the caller reloads the service manager before it starts anything.
|
||||||
|
func placeYieldSlice(r *declaration.Process) (bool, error) {
|
||||||
|
if r.Yields == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
path := filepath.Join(unitDir, declaration.YieldSlice)
|
||||||
|
want := yieldSlice(r.Yields)
|
||||||
|
if have, err := os.ReadFile(path); err == nil && string(have) == want {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
|
||||||
|
return false, fmt.Errorf("the yielding slice %s: %w", path, err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
// timerFor is the cadence a scheduled process runs on.
|
// timerFor is the cadence a scheduled process runs on.
|
||||||
//
|
//
|
||||||
// **The mesh's cron expression, handed to the machine's own timer.** The host already parses and
|
// **The mesh's cron expression, handed to the machine's own timer.** The host already parses and
|
||||||
@@ -446,7 +564,7 @@ func witnessOf(r *declaration.Process) string {
|
|||||||
if r.Witness != "" {
|
if r.Witness != "" {
|
||||||
return r.Witness
|
return r.Witness
|
||||||
}
|
}
|
||||||
return witness.Default(r.Name)
|
return witness.Default(r.ID, r.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// digestWritten is the bundle digest a process's record says was placed: the first half of what it
|
// digestWritten is the bundle digest a process's record says was placed: the first half of what it
|
||||||
@@ -460,3 +578,81 @@ func digestWritten(wrote string) string {
|
|||||||
|
|
||||||
// DaemonRoot is where unpacked daemons live — and what a witness keeps beside them.
|
// DaemonRoot is where unpacked daemons live — and what a witness keeps beside them.
|
||||||
func DaemonRoot() string { return daemonRoot }
|
func DaemonRoot() string { return daemonRoot }
|
||||||
|
|
||||||
|
// unitDifference names what differs between what a process's record says was written and what is wanted now, both
|
||||||
|
// "sha256:<hex> <unit>": its bundle when the digest differs, and in the unit file each directive and each environment
|
||||||
|
// word that differs, with inside a word the keys of a JSON object (and, one level down, of its objects) that differ. Names only, never a value, so a secret in a word is never said. Empty
|
||||||
|
// when nothing differs or there was no record.
|
||||||
|
func unitDifference(was, want string) string {
|
||||||
|
if was == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
wasDigest, wasUnit, _ := strings.Cut(was, " ")
|
||||||
|
wantDigest, wantUnit, _ := strings.Cut(want, " ")
|
||||||
|
var names []string
|
||||||
|
if wasDigest != wantDigest {
|
||||||
|
names = append(names, "its bundle")
|
||||||
|
}
|
||||||
|
before, after := unitDirectives(wasUnit), unitDirectives(wantUnit)
|
||||||
|
keys := map[string]bool{}
|
||||||
|
for k := range before {
|
||||||
|
keys[k] = true
|
||||||
|
}
|
||||||
|
for k := range after {
|
||||||
|
keys[k] = true
|
||||||
|
}
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(keys)) {
|
||||||
|
if before[k] != after[k] {
|
||||||
|
names = append(names, "its unit's "+k+valueDifference(before[k], after[k], 2))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitDirectives is a unit's directives by name: "Environment <word>" for an environment word, the directive's own
|
||||||
|
// name for the rest, each with its value as written (an environment word's value unescaped, as unitValue escapes it).
|
||||||
|
func unitDirectives(unit string) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, line := range strings.Split(unit, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "[") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, value, _ := strings.Cut(line, "=")
|
||||||
|
if key == "Environment" && strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) {
|
||||||
|
word, rest, _ := strings.Cut(value[1:len(value)-1], "=")
|
||||||
|
rest = strings.NewReplacer(`\\`, `\`, `\"`, `"`, "%%", "%").Replace(rest)
|
||||||
|
out["Environment "+word] = rest
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[key] += value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// valueDifference is what differs inside one directive's value, as " (a, b)": the keys of a JSON object whose value
|
||||||
|
// differs (going down depth levels), else the names of the name=value entries of a comma-separated list that were
|
||||||
|
// added, removed or changed. Empty when the value says nothing more than that it differs.
|
||||||
|
func valueDifference(was, now string, depth int) string {
|
||||||
|
var a, b map[string]json.RawMessage
|
||||||
|
if depth > 0 && json.Unmarshal([]byte(was), &a) == nil && json.Unmarshal([]byte(now), &b) == nil && (a != nil || b != nil) {
|
||||||
|
all := map[string]bool{}
|
||||||
|
for k := range a {
|
||||||
|
all[k] = true
|
||||||
|
}
|
||||||
|
for k := range b {
|
||||||
|
all[k] = true
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(all)) {
|
||||||
|
if string(a[k]) == string(b[k]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
inner := valueDifference(string(a[k]), string(b[k]), depth-1)
|
||||||
|
names = append(names, k+inner)
|
||||||
|
}
|
||||||
|
return " (" + strings.Join(names, ", ") + ")"
|
||||||
|
}
|
||||||
|
// Anything else is a value: said as differing, never split, since a name before an "=" in it can be a secret.
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
+117
-20
@@ -21,10 +21,16 @@ func aProcess() *declaration.Process {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The unit the mesh writes says what it runs, where, and that it comes back.
|
// The unit the mesh writes says what it runs, where, and that it comes back — and that it is the
|
||||||
|
// mesh's: **generated whole and saying so**, because an edit that survives until the next declaration
|
||||||
|
// and then vanishes is worse than one that is refused. The long-running mode stays up and is not a step.
|
||||||
func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
||||||
unit := unitFor(aProcess())
|
unit := unitFor(aProcess())
|
||||||
|
if !strings.HasPrefix(unit, "#") {
|
||||||
|
t.Fatalf("the unit does not open saying it is generated:\n%s", unit)
|
||||||
|
}
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
|
"Do not edit",
|
||||||
"ExecStart=node index.js",
|
"ExecStart=node index.js",
|
||||||
"WorkingDirectory=/var/lib/mesh/daemons/greeter",
|
"WorkingDirectory=/var/lib/mesh/daemons/greeter",
|
||||||
"Restart=always",
|
"Restart=always",
|
||||||
@@ -34,14 +40,8 @@ func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
|||||||
t.Fatalf("the unit does not say %q:\n%s", want, unit)
|
t.Fatalf("the unit does not say %q:\n%s", want, unit)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
if strings.Contains(unit, "Type=oneshot") {
|
||||||
|
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
||||||
// **Generated whole and saying so.** Every managed file on a machine carries this, because an edit
|
|
||||||
// that survives until the next declaration and then vanishes is worse than one that is refused.
|
|
||||||
func TestTheUnitSaysItIsTheMeshs(t *testing.T) {
|
|
||||||
unit := unitFor(aProcess())
|
|
||||||
if !strings.HasPrefix(unit, "#") || !strings.Contains(unit, "Do not edit") {
|
|
||||||
t.Fatalf("the unit does not say it is generated:\n%s", unit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,6 +83,60 @@ func TestAProcessRunsAsWhoItSaid(t *testing.T) {
|
|||||||
if strings.Contains(unitFor(aProcess()), "User=") {
|
if strings.Contains(unitFor(aProcess()), "User=") {
|
||||||
t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess()))
|
t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A process that yields (ADR 0553) runs in the node's one yielding slice at nice 19; the slice holds the idle weight
|
||||||
|
// and the quota for its share of the machine's processors; one that does not yield has none of it.
|
||||||
|
defer func(n func() int) { machineCPUs = n }(machineCPUs)
|
||||||
|
for _, c := range []struct {
|
||||||
|
cpus int
|
||||||
|
y *declaration.Yields
|
||||||
|
unit, slice, not []string
|
||||||
|
}{
|
||||||
|
{24, &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50},
|
||||||
|
[]string{"Slice=meshyield.slice\n", "Nice=19\n", "IOSchedulingPriority=7\n", "50%"},
|
||||||
|
[]string{"CPUWeight=idle\n", "IOWeight=1\n", "CPUQuota=1200%\n"}, []string{"CPUQuota=", "CPUWeight="}},
|
||||||
|
{1, &declaration.Yields{CPUShare: 25}, []string{"Slice=meshyield.slice\n"}, []string{"CPUQuota=100%\n"},
|
||||||
|
[]string{"Nice="}},
|
||||||
|
{24, nil, nil, nil, []string{"Slice=", "Nice=", "CPUQuota="}},
|
||||||
|
} {
|
||||||
|
machineCPUs = func() int { return c.cpus }
|
||||||
|
p := aProcess()
|
||||||
|
p.Yields = c.y
|
||||||
|
unit := unitFor(p)
|
||||||
|
for _, w := range c.unit {
|
||||||
|
if !strings.Contains(unit, w) {
|
||||||
|
t.Errorf("yields %+v: the unit does not say %q:\n%s", c.y, w, unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range c.not {
|
||||||
|
if strings.Contains(unit, n) {
|
||||||
|
t.Errorf("yields %+v: the unit says %q:\n%s", c.y, n, unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.y == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
slice := yieldSlice(c.y)
|
||||||
|
for _, w := range c.slice {
|
||||||
|
if !strings.Contains(slice, w) {
|
||||||
|
t.Errorf("yields %+v on %d processors: the slice does not say %q:\n%s", c.y, c.cpus, w, slice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The slice is placed beside the units, once, and again only when it differs.
|
||||||
|
was := unitDir
|
||||||
|
unitDir = t.TempDir()
|
||||||
|
defer func() { unitDir = was }()
|
||||||
|
p := aProcess()
|
||||||
|
p.Yields = &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50}
|
||||||
|
for i, want := range []bool{true, false} {
|
||||||
|
if wrote, err := placeYieldSlice(p); err != nil || wrote != want {
|
||||||
|
t.Errorf("placing %d: wrote %v (%v), want %v", i, wrote, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b, err := os.ReadFile(filepath.Join(unitDir, "meshyield.slice")); err != nil || !strings.Contains(string(b), "[Slice]") {
|
||||||
|
t.Errorf("the slice file: %q %v", b, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Three modes, one kind.** A scheduled process is a timer plus a unit that finishes, not a unit
|
// **Three modes, one kind.** A scheduled process is a timer plus a unit that finishes, not a unit
|
||||||
@@ -124,17 +178,6 @@ func TestACronBecomesATimersCalendar(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// And the long-running mode is unchanged by any of it: it stays up and comes back.
|
|
||||||
func TestAProcessThatStaysUpIsStillRestartedWhenItExits(t *testing.T) {
|
|
||||||
unit := unitFor(aProcess())
|
|
||||||
if !strings.Contains(unit, "Restart=always") {
|
|
||||||
t.Fatalf("a process that should stay up is not restarted when it exits:\n%s", unit)
|
|
||||||
}
|
|
||||||
if strings.Contains(unit, "Type=oneshot") {
|
|
||||||
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A unit file reinterprets a value in three ways nothing else does**, and a module's environment
|
// **A unit file reinterprets a value in three ways nothing else does**, and a module's environment
|
||||||
// routinely contains all three — a generated password is arbitrary bytes.
|
// routinely contains all three — a generated password is arbitrary bytes.
|
||||||
func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
||||||
@@ -276,11 +319,17 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
|
d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
|
||||||
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
|
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
|
||||||
|
d.For = "anchor"
|
||||||
|
|
||||||
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// Design 47 G A-342: the process runs with its node's name as MESH_NODE beside its own env.
|
||||||
|
if b, err := os.ReadFile(filepath.Join(units, "node-tools.service")); err != nil || !strings.Contains(string(b), `Environment="MESH_NODE=anchor"`) ||
|
||||||
|
!strings.Contains(string(b), `Environment="MESH_TOOL_MODULES=`) {
|
||||||
|
t.Errorf("the unit does not carry the node's name beside its env (%v):\n%s", err, b)
|
||||||
|
}
|
||||||
if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
|
if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
|
||||||
t.Fatalf("first apply: %+v, want created", o)
|
t.Fatalf("first apply: %+v, want created", o)
|
||||||
}
|
}
|
||||||
@@ -314,6 +363,54 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
|
|||||||
if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
|
if o := outcomeOf(third, "node-tools.runtime"); o.Action != "unchanged" {
|
||||||
t.Errorf("third apply: %+v, want unchanged", o)
|
t.Errorf("third apply: %+v, want unchanged", o)
|
||||||
}
|
}
|
||||||
|
// An input of its unit that differs restarts it, and the line says which (issue 1966).
|
||||||
|
changed := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
|
||||||
|
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js,b=/y/index.js"}}`)
|
||||||
|
changed.For = "anchor"
|
||||||
|
fourth, _, err := Apply(context.Background(), archHost(t), changed, state, store.OriginDeclared, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(fourth, "node-tools.runtime"); o.Action != "updated" || !strings.Contains(o.Detail, "its unit's Environment MESH_TOOL_MODULES") ||
|
||||||
|
strings.Contains(o.Detail, "/y/") {
|
||||||
|
t.Errorf("fourth apply: %+v, want updated and the word that changed named, no value", o)
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name, was, now, says string
|
||||||
|
}{
|
||||||
|
{"a word of a module inside a JSON word", `{"files":{"OWNED":"/a"},"mail":{"X":"1"}}`, `{"files":{"OWNED":"/b"},"mail":{"X":"1"}}`,
|
||||||
|
"Environment TOOL_ENV (files (OWNED))"},
|
||||||
|
{"a secret-shaped value", "c2VjcmV0MQ==", "c2VjcmV0Mg==", "Environment TOOL_VALUE"},
|
||||||
|
{"a URL with a password", "postgres://u:hunter1@h/db?sslmode=x", "postgres://u:hunter2@h/db?sslmode=x", "Environment TOOL_VALUE"},
|
||||||
|
{"nothing changed", "a=/x/1", "a=/x/1", ""},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
word := "TOOL_VALUE"
|
||||||
|
if strings.HasPrefix(row.was, "{") {
|
||||||
|
word = "TOOL_ENV"
|
||||||
|
}
|
||||||
|
r := aProcess()
|
||||||
|
mk := func(v string) string {
|
||||||
|
r.Env = map[string]string{word: v}
|
||||||
|
return "sha256:" + strings.Repeat("a", 64) + " " + unitFor(r)
|
||||||
|
}
|
||||||
|
got := unitDifference(mk(row.was), mk(row.now))
|
||||||
|
if row.says == "" {
|
||||||
|
if got != "" {
|
||||||
|
t.Fatalf("said %q for nothing changed", got)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, leak := range []string{"c2VjcmV0", "hunter", "sslmode", "postgres", "/b"} {
|
||||||
|
if strings.Contains(got, leak) {
|
||||||
|
t.Fatalf("said %q, which holds part of a value (%s)", got, leak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, row.says) {
|
||||||
|
t.Fatalf("said %q, want it to name %q", got, row.says)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
|
// novox/hq ADR 0193: a bundle compiled to a binary runs itself — `./name` is that file in the
|
||||||
|
|||||||
@@ -3,7 +3,9 @@ package apply
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -70,36 +72,112 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
|
|||||||
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||||
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||||
|
|
||||||
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
|
oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
|
||||||
newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
|
newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
|
||||||
controller := func(body []byte, digest, env string) string {
|
runtime := func(body []byte, digest, env string) string {
|
||||||
return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
|
return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
|
||||||
`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
|
`","digest":"` + digest + `","run":["./node-tools"],"env":{"X":"` + env + `"}}`
|
||||||
}
|
}
|
||||||
_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
|
_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest, "1")), store.State{},
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
|
_, state, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "1")), state,
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
|
if _, err := witness.Restore(context.Background(), bundles, "node-tools", "never answered", witness.Runner(run),
|
||||||
time.Now()); err != nil {
|
time.Now()); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// The mesh still declares the new build, and its unit changes: the process is re-placed.
|
// The mesh still declares the new build, and its unit changes: the process is re-placed.
|
||||||
report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
|
report, _, err := Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "2")), state,
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
|
if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "old\n" {
|
||||||
t.Fatalf("the rolled-back build was placed again: %q", got)
|
t.Fatalf("the rolled-back build was placed again: %q", got)
|
||||||
}
|
}
|
||||||
if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
|
if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "rolled back on this machine") {
|
||||||
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A process nobody judges keeps the directory it works in until its restart** (novox/hq issue 734).
|
||||||
|
// On 2026-10-26 three checks on one builder were refused in the second a new build of it arrived: the
|
||||||
|
// applier deleted the running build's directory before the restart, and the `go version` the old
|
||||||
|
// process started in it failed with "getwd: no such file or directory". Here a process works in the
|
||||||
|
// build's directory without running from it, as that `go version` did: the apply of a new build leaves
|
||||||
|
// its directory in place, and the retired build goes once nothing works in it.
|
||||||
|
func TestAProcessNobodyJudgesKeepsItsDirectoryUntilItsRestart(t *testing.T) {
|
||||||
|
if _, err := os.Stat("/proc/self/cwd"); err != nil {
|
||||||
|
t.Skip("which process works in a build is read from /proc")
|
||||||
|
}
|
||||||
|
units, bundles := t.TempDir(), t.TempDir()
|
||||||
|
wasUnits, wasBundles := unitDir, daemonRoot
|
||||||
|
unitDir, daemonRoot = units, bundles
|
||||||
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||||
|
|
||||||
|
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-build-go": "old\n"})
|
||||||
|
newBody, newDigest := anArchive(t, map[string]string{"mesh-build-go": "new\n"})
|
||||||
|
builder := func(body []byte, digest string) string {
|
||||||
|
return `{"id":"mesh-build-go.builder","type":"process","name":"mesh-build-go","source":"` + serving(t, body) +
|
||||||
|
`","digest":"` + digest + `","run":["./mesh-build-go"]}`
|
||||||
|
}
|
||||||
|
_, state, err := Apply(context.Background(), archHost(t), declare(t, builder(oldBody, oldDigest)), store.State{},
|
||||||
|
store.OriginDeclared, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sleep, err := exec.LookPath("sleep")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("no sleep to run")
|
||||||
|
}
|
||||||
|
working := exec.Command(sleep, "60")
|
||||||
|
working.Dir = filepath.Join(bundles, "mesh-build-go")
|
||||||
|
if err := working.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = working.Process.Kill(); _ = working.Wait() })
|
||||||
|
cwd := filepath.Join("/proc", strconv.Itoa(working.Process.Pid), "cwd")
|
||||||
|
for deadline := time.Now().Add(5 * time.Second); ; {
|
||||||
|
if at, err := os.Readlink(cwd); err == nil && at == working.Dir {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("PID %d does not work in %s within 5s", working.Process.Pid, working.Dir)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, _, err := Apply(context.Background(), archHost(t), declare(t, builder(newBody, newDigest)), state,
|
||||||
|
store.OriginDeclared, run, nil, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-build-go", "mesh-build-go")); string(got) != "new\n" {
|
||||||
|
t.Fatalf("the new build is not where the unit runs it from: %q", got)
|
||||||
|
}
|
||||||
|
at, err := os.Readlink(cwd)
|
||||||
|
if err != nil || strings.HasSuffix(at, " (deleted)") {
|
||||||
|
t.Fatalf("the directory a running process works in was deleted before its restart: %q (%v)", at, err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(filepath.Join(at, "mesh-build-go")); string(got) != "old\n" {
|
||||||
|
t.Fatalf("the process's directory no longer holds its own build: %q", got)
|
||||||
|
}
|
||||||
|
if len(witness.Trials(bundles)) != 0 {
|
||||||
|
t.Fatal("a process nobody judges went on trial")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Its process gone, the next look deletes the retired build and keeps nothing about the process.
|
||||||
|
_ = working.Process.Kill()
|
||||||
|
_ = working.Wait()
|
||||||
|
witness.SweepAll(bundles)
|
||||||
|
if _, err := os.Stat(witness.Dir(bundles, "mesh-build-go")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("a retired build nothing works in was kept: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
|
|||||||
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
|
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
|
||||||
// once however many files name its digest.
|
// once however many files name its digest.
|
||||||
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
|
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
|
||||||
|
return ResolveFileReferencesKept(ctx, d, fetch, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
|
||||||
|
// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
|
||||||
|
// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
|
||||||
|
// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
|
||||||
|
// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
|
||||||
|
// own names objects the controller no longer knows of). A nil keep keeps nothing.
|
||||||
|
func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
|
||||||
fetched := map[string][]byte{}
|
fetched := map[string][]byte{}
|
||||||
for i, r := range d.Resources {
|
for i, r := range d.Resources {
|
||||||
f, ok := r.(*declaration.File)
|
f, ok := r.(*declaration.File)
|
||||||
if !ok || !f.IsReference() {
|
if !ok || !f.IsReference() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
body, err := referencedContent(ctx, f, fetch, fetched)
|
body, err := referencedContent(ctx, f, fetch, fetched, keep)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
|
|||||||
}
|
}
|
||||||
|
|
||||||
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
|
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
|
||||||
fetched map[string][]byte) ([]byte, error) {
|
fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
|
||||||
if body, ok := fetched[f.Digest]; ok {
|
if body, ok := fetched[f.Digest]; ok {
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
if body, ok := alreadyThere(f); ok {
|
|
||||||
fetched[f.Digest] = body
|
|
||||||
return body, nil
|
|
||||||
}
|
|
||||||
name, err := declaration.ObjectName(f.Digest)
|
name, err := declaration.ObjectName(f.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
|
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
|
||||||
}
|
}
|
||||||
|
if body, ok := alreadyThere(f); ok {
|
||||||
|
fetched[f.Digest] = body
|
||||||
|
keep.put(name, body)
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
if body, ok := keep.get(name, f); ok {
|
||||||
|
fetched[f.Digest] = body
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
if fetch == nil {
|
if fetch == nil {
|
||||||
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
|
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
|
||||||
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
|
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
|
||||||
}
|
}
|
||||||
body, err := fetch(ctx, name, f.Size)
|
body, err := fetch(ctx, name, f.Size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
|
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
|
||||||
f.ID, f.Path, name, err)
|
f.ID, f.Path, name, keep.never(), err)
|
||||||
}
|
}
|
||||||
if err := f.Verify(body); err != nil {
|
if err := f.Verify(body); err != nil {
|
||||||
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
|
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
|
||||||
}
|
}
|
||||||
fetched[f.Digest] = body
|
fetched[f.Digest] = body
|
||||||
|
keep.put(name, body)
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -118,10 +118,10 @@ func TestAFileIsJudgedByTheSameGuardAsADirectory(t *testing.T) {
|
|||||||
if _, statErr := os.Stat(filepath.Join(home, "data", "notes.conf")); statErr == nil {
|
if _, statErr := os.Stat(filepath.Join(home, "data", "notes.conf")); statErr == nil {
|
||||||
t.Error("a file was written below a directory the engine refused")
|
t.Error("a file was written below a directory the engine refused")
|
||||||
}
|
}
|
||||||
// A refused file has no outcome: it is a failure. Declared first, it is the failure the apply reports,
|
// A refused file has no outcome: it is a failure. Declared first, it is the failure the apply reports, and the
|
||||||
// and the directory the other.
|
// directory after it is left unapplied, as the rest of its module is (design 42 zfs-owns-zfs §1).
|
||||||
var applyErr *Error
|
var applyErr *Error
|
||||||
if !errors.As(err, &applyErr) || applyErr.Resource != "notes.conf" || applyErr.Others != 1 {
|
if !errors.As(err, &applyErr) || applyErr.Resource != "notes.conf" || applyErr.Others != 0 {
|
||||||
t.Errorf("the file below a refused directory is not among the failures: %v", err)
|
t.Errorf("the file below a refused directory is not among the failures: %v", err)
|
||||||
}
|
}
|
||||||
if o := outcomeOf(report, "notes.conf"); o.ID != "" {
|
if o := outcomeOf(report, "notes.conf"); o.ID != "" {
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package apply
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -47,6 +49,162 @@ func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testi
|
|||||||
if !strings.Contains(err.Error(), pinned) {
|
if !strings.Contains(err.Error(), pinned) {
|
||||||
t.Errorf("the failure does not name the image %s: %v", pinned, err)
|
t.Errorf("the failure does not name the image %s: %v", pinned, err)
|
||||||
}
|
}
|
||||||
|
var pull *PullError
|
||||||
|
if !errors.As(err, &pull) || pull.Image != pinned {
|
||||||
|
t.Errorf("the failure is not a pull error naming %s: %v", pinned, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// design 51, ENG-7: a module of two containers whose second new image cannot be pulled replaces neither, and says the
|
||||||
|
// image; with both pullable, both are pulled before the first is set aside.
|
||||||
|
second := strings.Replace(pinned, "postgres", "other", 1)
|
||||||
|
two := `{"declaration":1,"resources":[
|
||||||
|
{"id":"a.first","type":"container","name":"first","image":"` + pinned + `","env":{"X":"1"}},
|
||||||
|
{"id":"a.second","type":"container","name":"second","image":"` + second + `","env":{"X":"2"}}
|
||||||
|
]}`
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
unpullable string
|
||||||
|
wantTouched bool
|
||||||
|
wantPulled string
|
||||||
|
}{
|
||||||
|
{"the second image cannot be pulled", second, false, "pull " + pinned},
|
||||||
|
{"both can", "", true, "pull " + pinned + "; pull " + second},
|
||||||
|
} {
|
||||||
|
d := parseTrusted(t, two)
|
||||||
|
var steps []string
|
||||||
|
present := map[string]bool{}
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
args = oldShape(name, args)
|
||||||
|
switch args[0] {
|
||||||
|
case "info":
|
||||||
|
return "27.0\n", nil
|
||||||
|
case "container":
|
||||||
|
return "true\tsome-older-spec", nil
|
||||||
|
case "image":
|
||||||
|
if present[args[len(args)-1]] {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return "", errors.New("no such image")
|
||||||
|
case "pull":
|
||||||
|
if args[1] == c.unpullable {
|
||||||
|
return "", errors.New("manifest unknown")
|
||||||
|
}
|
||||||
|
present[args[1]] = true
|
||||||
|
steps = append(steps, "pull "+args[1])
|
||||||
|
case "rm", "rename", "stop", "run":
|
||||||
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||||
|
got := strings.Join(steps, "; ")
|
||||||
|
touched := strings.Contains(got, "rename") || strings.Contains(got, "run ")
|
||||||
|
if touched != c.wantTouched || !strings.HasPrefix(got, c.wantPulled) {
|
||||||
|
t.Errorf("%s: steps %q (touched %v), want pulls %q first and touched=%v", c.name, got, touched, c.wantPulled, c.wantTouched)
|
||||||
|
}
|
||||||
|
if !c.wantTouched && (err == nil || !strings.Contains(err.Error(), second)) {
|
||||||
|
t.Errorf("%s: the failure does not name %s: %v", c.name, second, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1966 (2026-11-08 23:25 UTC): a module whose container serves the artifact store had it replaced, and then
|
||||||
|
// could not fetch its program from the store it had cut off. Every archive and process is fetched before anything of
|
||||||
|
// the module is replaced or removed: with the store away, its container and its orphan stay as they were, and the
|
||||||
|
// failure names the program's source.
|
||||||
|
t.Run("an artifact that cannot be fetched replaces nothing of its module", func(t *testing.T) {
|
||||||
|
gone := httptest.NewServer(http.NotFoundHandler())
|
||||||
|
source := gone.URL + "/program.tar.gz"
|
||||||
|
gone.Close() // the store is cut off: refused
|
||||||
|
was := storeAwayAtMost
|
||||||
|
storeAwayAtMost = 0 // no patience: the refusal fails at once, without waiting
|
||||||
|
t.Cleanup(func() { storeAwayAtMost = was })
|
||||||
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"distribution.store","type":"container","name":"store","image":"`+pinned+`","env":{"X":"new"}},
|
||||||
|
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
|
||||||
|
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
|
||||||
|
]}`)
|
||||||
|
known := store.State{}
|
||||||
|
known.Record(store.Applied{ID: "distribution.old", Type: string(declaration.TypeContainer), Target: "old",
|
||||||
|
Module: "distribution"})
|
||||||
|
var steps []string
|
||||||
|
removed := map[string]bool{}
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
if name == "systemctl" {
|
||||||
|
steps = append(steps, "systemctl "+strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
args = oldShape(name, args)
|
||||||
|
switch args[0] {
|
||||||
|
case "info":
|
||||||
|
return "27.0\n", nil
|
||||||
|
case "container":
|
||||||
|
if removed[args[len(args)-1]] {
|
||||||
|
return "", errors.New("no such container")
|
||||||
|
}
|
||||||
|
return "true\tsome-older-spec", nil
|
||||||
|
case "rm", "rename", "stop", "run":
|
||||||
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
||||||
|
if args[0] == "rm" {
|
||||||
|
removed[args[len(args)-1]] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
||||||
|
if len(steps) > 0 {
|
||||||
|
t.Errorf("the module was touched though its program could not be fetched: %q", steps)
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), source) {
|
||||||
|
t.Errorf("the failure does not name the program's source %s: %v", source, err)
|
||||||
|
}
|
||||||
|
if _, still := after.Find("distribution.old"); !still {
|
||||||
|
t.Error("the module's orphan was removed though its program could not be fetched")
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "distribution.store"); o.Action != "skipped" {
|
||||||
|
t.Errorf("the container's outcome is %+v, want skipped", o)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Design 47 (W13.8): the holder's own program is fetched from the build node that kept it, not from the store; when it
|
||||||
|
// cannot be had, or is not the declared digest, nothing of the module is replaced and the failure says so.
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
fetch KeptFetch
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{"the node that keeps it is away", func(context.Context, string, string) ([]byte, bool, error) {
|
||||||
|
return nil, true, errors.New("novox keeps it and does not answer")
|
||||||
|
}, "novox keeps it and does not answer"},
|
||||||
|
{"what the node kept is not the declared digest", func(context.Context, string, string) ([]byte, bool, error) {
|
||||||
|
return []byte("other"), true, nil
|
||||||
|
}, "what arrived is"},
|
||||||
|
} {
|
||||||
|
t.Run("a kept artifact: "+row.name, func(t *testing.T) {
|
||||||
|
gone := httptest.NewServer(http.NotFoundHandler())
|
||||||
|
source := gone.URL + "/program.tar.gz"
|
||||||
|
gone.Close()
|
||||||
|
was := storeAwayAtMost
|
||||||
|
storeAwayAtMost = 0
|
||||||
|
t.Cleanup(func() { storeAwayAtMost = was })
|
||||||
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
|
||||||
|
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
|
||||||
|
]}`)
|
||||||
|
var asked []string
|
||||||
|
fetch := func(ctx context.Context, module, digest string) ([]byte, bool, error) {
|
||||||
|
asked = append(asked, module+" "+digest)
|
||||||
|
return row.fetch(ctx, module, digest)
|
||||||
|
}
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||||
|
_, _, err := Apply(WithKept(context.Background(), fetch), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), row.wantErr) {
|
||||||
|
t.Errorf("the failure is %v, want it to say %q", err, row.wantErr)
|
||||||
|
}
|
||||||
|
if len(asked) != 1 || asked[0] != "distribution sha256:"+strings.Repeat("0", 64) {
|
||||||
|
t.Errorf("the kept fetch was asked %q", asked)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
|
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
|
||||||
@@ -54,6 +212,16 @@ func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing
|
|||||||
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
||||||
]}`)
|
]}`)
|
||||||
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
||||||
|
// The seat and holder a routed image names (ENG-10) are no part of what a container is: another holder is the same spec,
|
||||||
|
// so a declaration composed again for it replaces no container.
|
||||||
|
for _, holder := range []string{"distribution", "elsewhere"} {
|
||||||
|
named := parseTrusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","image-seat":"mesh-artifact-store","image-holder":"`+holder+`","env":{"PGDATA":"/data"}}
|
||||||
|
]}`)
|
||||||
|
if got := containerSpec(named.Resources[0].(*declaration.Container), inputs{}); got != want {
|
||||||
|
t.Errorf("a container naming the holder %s has another spec than one naming none", holder)
|
||||||
|
}
|
||||||
|
}
|
||||||
var order []string
|
var order []string
|
||||||
created := false
|
created := false
|
||||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StopModule stops the units and containers the node's record holds for a module and removes nothing: no unit, file,
|
||||||
|
// container, data or record, so what the module kept survives and a later Apply starts the build it is given (design 47
|
||||||
|
// G A-98, §297). Units come first, a timer before the service it starts, then containers; a unit the mesh never held
|
||||||
|
// (Stateless) is left as it is. It answers what it stopped, and the first failure.
|
||||||
|
func StopModule(ctx context.Context, sys system.System, known store.State, module string, run Runner) ([]string, error) {
|
||||||
|
var timers, services, containers []store.Applied
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if store.OwnerOf(r, declaration.ModuleGuess) != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch r.Type {
|
||||||
|
case string(declaration.TypeService):
|
||||||
|
if r.Stateless {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(r.Target, ".timer") {
|
||||||
|
timers = append(timers, r)
|
||||||
|
} else {
|
||||||
|
services = append(services, r)
|
||||||
|
}
|
||||||
|
case string(declaration.TypeContainer):
|
||||||
|
containers = append(containers, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var stopped []string
|
||||||
|
for _, r := range append(timers, services...) {
|
||||||
|
manager := managerFor(r.Scope, r.User, run)
|
||||||
|
if state, err := sys.ServiceState(ctx, manager, r.Target); err != nil {
|
||||||
|
if strings.Contains(err.Error(), "does not exist on this machine") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err)
|
||||||
|
} else if state == "stopped" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := sys.SetServiceState(ctx, manager, r.Target, "stopped"); err != nil {
|
||||||
|
return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err)
|
||||||
|
}
|
||||||
|
stopped = append(stopped, r.Target)
|
||||||
|
}
|
||||||
|
if len(containers) == 0 {
|
||||||
|
return stopped, nil
|
||||||
|
}
|
||||||
|
cri, err := containerRuntime(ctx, run)
|
||||||
|
if err != nil {
|
||||||
|
return stopped, err
|
||||||
|
}
|
||||||
|
sort.Slice(containers, func(i, j int) bool { return containers[i].Target < containers[j].Target })
|
||||||
|
for _, r := range containers {
|
||||||
|
if _, err := run(ctx, cri, "stop", r.Target); err != nil {
|
||||||
|
return stopped, fmt.Errorf("stopping the container %s of %s: %w", r.Target, module, err)
|
||||||
|
}
|
||||||
|
stopped = append(stopped, r.Target)
|
||||||
|
}
|
||||||
|
return stopped, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Stop stops a module's timer, then its service, then its container, and leaves every record, and another module's
|
||||||
|
// units, as they are (design 47 G A-98).
|
||||||
|
func TestStopModuleStopsAModulesUnitsAndContainersAndRemovesNothing(t *testing.T) {
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{
|
||||||
|
"app.service": {active: "active", enabled: "enabled"},
|
||||||
|
"app.timer": {active: "active", enabled: "enabled"},
|
||||||
|
"other.service": {active: "active", enabled: "enabled"},
|
||||||
|
})
|
||||||
|
state := store.State{}
|
||||||
|
for _, r := range []store.Applied{
|
||||||
|
{ID: "app.run", Type: "service", Target: "app.service"},
|
||||||
|
{ID: "app.tick", Type: "service", Target: "app.timer"},
|
||||||
|
{ID: "app.c", Type: "container", Target: "app-box"},
|
||||||
|
{ID: "other.run", Type: "service", Target: "other.service"},
|
||||||
|
{ID: "other.c", Type: "container", Target: "other-box"},
|
||||||
|
} {
|
||||||
|
state.Record(r)
|
||||||
|
}
|
||||||
|
got, err := StopModule(context.Background(), archHost(t), state, "app", m.run)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := []string{"app.timer", "app.service", "app-box"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("stopped %v, want %v, the timer first", got, want)
|
||||||
|
}
|
||||||
|
if m.units["app.service"].active != "inactive" || m.units["app.timer"].active != "inactive" || m.units["other.service"].active != "active" {
|
||||||
|
t.Fatalf("units are %+v", m.units)
|
||||||
|
}
|
||||||
|
if !m.did("docker stop app-box") || m.did("docker stop other-box") || m.did("docker rm") {
|
||||||
|
t.Fatalf("the machine was asked %v", m.asked)
|
||||||
|
}
|
||||||
|
if len(state.Resources) != 5 {
|
||||||
|
t.Fatalf("a record was removed: %d left", len(state.Resources))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
// Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests,
|
||||||
|
// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449).
|
||||||
|
//
|
||||||
|
// Copied from mesh-controller internal/beside at commit 6c2a8d62 (R4b, issue 1379); the one change is that Dir
|
||||||
|
// lays out the captured copy of a repository once per process, not once per test (see laidOut). In mesh-sdk the
|
||||||
|
// catalogue's tests read mesh-catalog and mesh-host only; the captured copy holds just what they read.
|
||||||
|
//
|
||||||
|
// A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on
|
||||||
|
// whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none
|
||||||
|
// was beside it skipped and said nothing. Now there are two inputs, both named, and no third:
|
||||||
|
//
|
||||||
|
// - In a merge check, the build seat clones each core repository beside the one checked (the catalogue
|
||||||
|
// at its main, the node-engine at the commit the mesh runs) and says where in MESH_CHECK_BESIDE. A
|
||||||
|
// test judges against those clones, so agreement with the other repository is checked where
|
||||||
|
// `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a
|
||||||
|
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
|
||||||
|
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
|
||||||
|
// go.mod pins. Each repository's tests read these siblings, and the checker places exactly these
|
||||||
|
// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog,
|
||||||
|
// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine
|
||||||
|
// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and
|
||||||
|
// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and
|
||||||
|
// mesh-catalog. In a mesh where either module has no source repository nothing is
|
||||||
|
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
|
||||||
|
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
|
||||||
|
// catalogue's main, not the group's head.
|
||||||
|
// - Anywhere else, the test judges against the copy captured in this repository's testdata/beside, at the
|
||||||
|
// commit testdata/beside/CAPTURED names. Never against a developer's own checkout: to judge one, set
|
||||||
|
// MESH_CHECK_BESIDE to the directory holding it, as the check does.
|
||||||
|
//
|
||||||
|
// The captured manifests are named module.json.captured, and put back as module.json in a directory of
|
||||||
|
// the test's own: a module.json anywhere in this repository is a module of it to the forge's announcer and
|
||||||
|
// the planner, and a merge would build and register a hundred copies of the catalogue's.
|
||||||
|
package beside
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Env is where a merge check says the repositories cloned beside the one checked are (internal/builder's
|
||||||
|
// EnvBeside, repeated here so a test does not import the builder).
|
||||||
|
const Env = "MESH_CHECK_BESIDE"
|
||||||
|
|
||||||
|
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
||||||
|
const CapturedSuffix = ".captured"
|
||||||
|
|
||||||
|
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
||||||
|
// says which in the test's log.
|
||||||
|
func Dir(t testing.TB, repository string) string {
|
||||||
|
t.Helper()
|
||||||
|
if root := os.Getenv(Env); root != "" {
|
||||||
|
dir := filepath.Join(root, repository)
|
||||||
|
if _, err := os.Stat(dir); err != nil {
|
||||||
|
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
||||||
|
"judged here: %v", repository, Env, root, err)
|
||||||
|
}
|
||||||
|
t.Logf("judged against %s as cloned beside this check", dir)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
from := filepath.Join(Captured(), repository)
|
||||||
|
if _, err := os.Stat(from); err != nil {
|
||||||
|
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
||||||
|
}
|
||||||
|
dir, err := layOut(repository, from)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
||||||
|
}
|
||||||
|
t.Logf("judged against the copy of %s captured in testdata/beside (see CAPTURED); a merge check "+
|
||||||
|
"judges it against the repository's own clone", repository)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// laidOut is the captured copies already put back in place in this process, by repository. A copy is
|
||||||
|
// read-only to every test, so it is laid out once per process and not once per test: with a copy per test
|
||||||
|
// the catalogue's tests alone took 2.14 s, over mesh-sdk's unit bound of 2 s, and with one 0.81 s (measured
|
||||||
|
// 2026-10-20). The directories are in the temp directory, which the system clears; a process cannot remove
|
||||||
|
// them while another test may still read them.
|
||||||
|
var (
|
||||||
|
laidOutMu sync.Mutex
|
||||||
|
laidOut = map[string]string{}
|
||||||
|
)
|
||||||
|
|
||||||
|
func layOut(repository, from string) (string, error) {
|
||||||
|
laidOutMu.Lock()
|
||||||
|
defer laidOutMu.Unlock()
|
||||||
|
if dir, ok := laidOut[repository]; ok {
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
|
root, err := os.MkdirTemp("", "beside-")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
dir := filepath.Join(root, repository)
|
||||||
|
err = filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, _ := filepath.Rel(from, path)
|
||||||
|
into := filepath.Join(dir, strings.TrimSuffix(rel, CapturedSuffix))
|
||||||
|
if d.IsDir() {
|
||||||
|
return os.MkdirAll(into, 0o755)
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(into, raw, 0o644)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
os.RemoveAll(root)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
laidOut[repository] = dir
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Catalogue is the catalogue's modules directory, as Dir finds the catalogue.
|
||||||
|
func Catalogue(t testing.TB) string {
|
||||||
|
t.Helper()
|
||||||
|
return filepath.Join(Dir(t, "mesh-catalog"), "modules")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Captured is this repository's testdata/beside, found from this file rather than from the working
|
||||||
|
// directory, so a test in any package reaches it.
|
||||||
|
func Captured() string {
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
return filepath.Join(filepath.Dir(file), "..", "..", "testdata", "beside")
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||||
@@ -185,7 +185,7 @@ func (r Rewritten) declares(id string) bool {
|
|||||||
// container runtime's configuration whole — and neither is anything the operator installs later.
|
// container runtime's configuration whole — and neither is anything the operator installs later.
|
||||||
var genesisTakes = map[string]bool{
|
var genesisTakes = map[string]bool{
|
||||||
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
|
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
|
||||||
"postgres": true, BusModule: true, "mesh-vault": true, "mesh-catalog": true,
|
"postgres": true, BusModule: true, "mesh-vault": true,
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
|
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
|
||||||
|
|||||||
@@ -64,7 +64,6 @@ const (
|
|||||||
StepStore Step = "store"
|
StepStore Step = "store"
|
||||||
StepBroker Step = "broker"
|
StepBroker Step = "broker"
|
||||||
StepVault Step = "vault"
|
StepVault Step = "vault"
|
||||||
StepCatalogue Step = "catalogue"
|
|
||||||
StepNetwork Step = "network"
|
StepNetwork Step = "network"
|
||||||
StepFilter Step = "filter"
|
StepFilter Step = "filter"
|
||||||
StepExtras Step = "extras"
|
StepExtras Step = "extras"
|
||||||
@@ -89,7 +88,7 @@ var Steps = []Step{
|
|||||||
// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
|
// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
|
||||||
// build, to say what it holds, on its network, filtering. They used to be things somebody
|
// build, to say what it holds, on its network, filtering. They used to be things somebody
|
||||||
// typed afterwards, which is how they went missing without anything complaining.
|
// typed afterwards, which is how they went missing without anything complaining.
|
||||||
StepBase, StepStore, StepBroker, StepVault, StepCatalogue, StepNetwork, StepFilter, StepExtras, StepExport,
|
StepBase, StepStore, StepBroker, StepVault, StepNetwork, StepFilter, StepExtras, StepExport,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Error is a failure, named by the step it happened in.
|
// Error is a failure, named by the step it happened in.
|
||||||
@@ -184,7 +183,7 @@ type Options struct {
|
|||||||
// SDKSource is where the mesh's shared library is built from. It is published to the package
|
// SDKSource is where the mesh's shared library is built from. It is published to the package
|
||||||
// registry before the base is built, because the base resolves it by version (novox/hq ADR 0076).
|
// registry before the base is built, because the base resolves it by version (novox/hq ADR 0076).
|
||||||
SDKSource Source
|
SDKSource Source
|
||||||
// Site is where this machine sits, for the private network's placement.
|
// Site is where this machine sits, for the private network's assignment.
|
||||||
Site string
|
Site string
|
||||||
// Answers are the choices, answered by flag: name → answer. What a terminal would be asked.
|
// Answers are the choices, answered by flag: name → answer. What a terminal would be asked.
|
||||||
Answers map[string]string
|
Answers map[string]string
|
||||||
@@ -750,12 +749,6 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
|||||||
return result, failed(StepVault, err)
|
return result, failed(StepVault, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- 15. catalogue --------------------------------------------------------------------
|
|
||||||
say("catalogue — the module graph: what is held, what a change reaches, what to rebuild")
|
|
||||||
if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil {
|
|
||||||
return result, failed(StepCatalogue, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- 16. network ----------------------------------------------------------------------
|
// ---- 16. network ----------------------------------------------------------------------
|
||||||
say("network — the private network, and this machine's name on it")
|
say("network — the private network, and this machine's name on it")
|
||||||
if err := PlaceOnTheNetwork(ctx, o, permanentControl, rewritten.BrokerAddress, say); err != nil {
|
if err := PlaceOnTheNetwork(ctx, o, permanentControl, rewritten.BrokerAddress, say); err != nil {
|
||||||
@@ -822,7 +815,6 @@ func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.Syste
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Published{}, failed(StepRegistry, err)
|
return Published{}, failed(StepRegistry, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- 8. publish -----------------------------------------------------------------------
|
// ---- 8. publish -----------------------------------------------------------------------
|
||||||
say("publish — the control plane's image gets its first manifest digest")
|
say("publish — the control plane's image gets its first manifest digest")
|
||||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||||
|
|||||||
@@ -69,8 +69,9 @@ func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *decl
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// meshBuiltImage is an image the mesh built, as it records it: `artifact-store://<module>/<artifact>@sha256:…`.
|
// meshBuiltImage is an image the mesh built, as it records it in either form it reads (ADR 0605 §9, issue 1966 ENG-4):
|
||||||
var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
|
// `artifact-store://<module>/<artifact>@sha256:…`, or the registry seat's `container-image-registry/<module>/<artifact>@sha256:…`.
|
||||||
|
var meshBuiltImage = regexp.MustCompile(`(?:artifact-store://|container-image-registry/)([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
|
||||||
|
|
||||||
// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this
|
// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this
|
||||||
// mesh's registry: the image that carries the snapshot program.
|
// mesh's registry: the image that carries the snapshot program.
|
||||||
|
|||||||
@@ -139,4 +139,12 @@ func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) {
|
|||||||
if got, want := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image, repository+"@"+digest; got != want {
|
if got, want := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image, repository+"@"+digest; got != want {
|
||||||
t.Errorf("the bundle raises the bus on %s; the nats module's server is %s (%s)", got, want, from)
|
t.Errorf("the bundle raises the bus on %s; the nats module's server is %s (%s)", got, want, from)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The image the mesh built is read in either form it is recorded in (ENG-4).
|
||||||
|
sum := strings.Repeat("c", 64)
|
||||||
|
for _, said := range []string{"artifact-store://nats/server@sha256:" + sum, "container-image-registry/nats/server@sha256:" + sum} {
|
||||||
|
if m := meshBuiltImage.FindStringSubmatch("build 1: " + said); m == nil || m[1] != "nats/server@sha256:"+sum {
|
||||||
|
t.Errorf("%s is read as %v", said, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -237,35 +237,3 @@ func TestAFileNothingOfTheControllerWritesIsStillRefused(t *testing.T) {
|
|||||||
t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err)
|
t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The controller's own manifest, as the mesh runs it, passes step 9 (issue 597): in a merge check the
|
|
||||||
// node-engine's check has the controller cloned beside it (MESH_CHECK_BESIDE), where it must be; else a
|
|
||||||
// checkout beside this one; else it skips aloud.
|
|
||||||
//
|
|
||||||
// The checker places the controller beside this repository (mesh-delivery's besideRules, since R4g, issue 1295), so a
|
|
||||||
// MESH_CHECK_BESIDE without a mesh-controller directory, or without its module.json, fails. Only a run with no
|
|
||||||
// MESH_CHECK_BESIDE and no checkout beside this one (a developer's clone alone) skips aloud.
|
|
||||||
func TestTheControllersOwnManifestPassesStepNine(t *testing.T) {
|
|
||||||
var manifest []byte
|
|
||||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
|
||||||
if info, err := os.Stat(filepath.Join(root, "mesh-controller")); err != nil || !info.IsDir() {
|
|
||||||
t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: the checker places mesh-controller beside the node-engine's check (mesh-delivery besideRules, issue 1295), and this test judges the controller's own manifest", root)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(filepath.Join(root, "mesh-controller", "module.json"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: %v", root, err)
|
|
||||||
}
|
|
||||||
manifest = raw
|
|
||||||
} else if raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-controller", "module.json")); err == nil {
|
|
||||||
manifest = raw
|
|
||||||
} else {
|
|
||||||
t.Skip("NOT JUDGED: no controller beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
|
|
||||||
}
|
|
||||||
wanted, err := theStepNineSecretsOf(t, manifest)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("genesis's step 9 refuses the controller the mesh runs: %v", err)
|
|
||||||
}
|
|
||||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" {
|
|
||||||
t.Errorf("step 9 delivers %v", wanted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,14 +13,24 @@ import (
|
|||||||
// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first".
|
// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first".
|
||||||
|
|
||||||
// genesisModules is every catalogue module genesis installs on its own, read from the steps that
|
// genesisModules is every catalogue module genesis installs on its own, read from the steps that
|
||||||
// install them: the registry, the build agent, the store, the bus, the vault, the catalogue and the
|
// install them: the registry, the build agent, the store, the bus, the vault and the
|
||||||
// filter (extras are the operator's choice and come after the base too, but are not genesis's own).
|
// filter (extras are the operator's choice and come after the base too, but are not genesis's own).
|
||||||
var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "mesh-catalog", "nftables"}
|
var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "nftables"}
|
||||||
|
|
||||||
// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real
|
// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real
|
||||||
// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a
|
// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a
|
||||||
// genesis.
|
// genesis.
|
||||||
func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) {
|
func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) {
|
||||||
|
// ADR 0604 section 7: the catalog is the indexer's seat now, so genesis has no catalogue step and installs or takes no
|
||||||
|
// mesh-catalog module.
|
||||||
|
for _, step := range Steps {
|
||||||
|
if step == "catalogue" {
|
||||||
|
t.Fatal("genesis still has a catalogue step")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if genesisTakes["mesh-catalog"] {
|
||||||
|
t.Fatal("an adopted genesis still takes mesh-catalog")
|
||||||
|
}
|
||||||
built := map[string]bool{}
|
built := map[string]bool{}
|
||||||
for _, tc := range genesisToolchains {
|
for _, tc := range genesisToolchains {
|
||||||
built[tc.Language] = true
|
built[tc.Language] = true
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane,
|
|||||||
//
|
//
|
||||||
// **Assigning is not being on the network** — a lesson paid for: the module installed, the names
|
// **Assigning is not being on the network** — a lesson paid for: the module installed, the names
|
||||||
// file was written with no names in it, and everything reported success, because nobody had said
|
// file was written with no names in it, and everything reported success, because nobody had said
|
||||||
// where this machine IS. So placement is part of the step, not a separate act.
|
// where this machine IS. So assignment is part of the step, not a separate act.
|
||||||
func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
|
func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
|
||||||
brokerAddress string, say func(string)) error {
|
brokerAddress string, say func(string)) error {
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user