Compare commits
886
Commits
03031a46dd
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e89a5defd | ||
|
|
4b11f66b4b | ||
|
|
0439af3649 | ||
|
|
5cc26e2659 | ||
|
|
91dc064f92 | ||
|
|
7b6468e2bb | ||
|
|
0cc1728a04 | ||
|
|
d1768241dc | ||
|
|
cdd8bd3cba | ||
|
|
121a8090b3 | ||
|
|
ba9d4787f9 | ||
|
|
87cfae48b2 | ||
|
|
179a4a6402 | ||
|
|
7bfeb466bf | ||
|
|
e7803f8943 | ||
|
|
baeb87d195 | ||
|
|
9172eccbc4 | ||
|
|
07ea73bd64 | ||
|
|
d57e2325f6 | ||
|
|
43c812e98e | ||
|
|
e3d994fde1 | ||
|
|
eb89cb7da3 | ||
|
|
ed7cefb5d8 | ||
|
|
f21b8c202d | ||
|
|
b4bf772de4 | ||
|
|
c5f76cb974 | ||
|
|
8700aabba3 | ||
|
|
06781d4dbc | ||
|
|
d0b208b668 | ||
|
|
85ad1eef74 | ||
|
|
51b89964c5 | ||
|
|
1a6a380507 | ||
|
|
0612916af0 | ||
|
|
10a6824abe | ||
|
|
bb285f9c1b | ||
|
|
cbed32524a | ||
|
|
b132653fc0 | ||
|
|
523ad97e28 | ||
|
|
5b4484efd8 | ||
|
|
0a43489a74 | ||
|
|
dbb77efab1 | ||
|
|
1b9f9355b5 | ||
|
|
72385fd84a | ||
|
|
3bb895152f | ||
|
|
c6d893e141 | ||
|
|
310ff0cabc | ||
|
|
13fcfae051 | ||
|
|
f8f33e54a7 | ||
|
|
869d15e6d3 | ||
|
|
292aa54bbb | ||
|
|
f539cdb7ec | ||
|
|
3b365ba7cc | ||
|
|
6eb41a17e1 | ||
|
|
a1cd4a1a2f | ||
|
|
7a30c0d54c | ||
|
|
a9eb4372d9 | ||
|
|
c025bb5f1e | ||
|
|
5a3510b115 | ||
|
|
7f892a757c | ||
|
|
40ed0633e0 | ||
|
|
5bbde5db40 | ||
|
|
21e452df28 | ||
|
|
dc32afdd7e | ||
|
|
8a0c39280d | ||
|
|
a00cccfe0d | ||
|
|
592b4002bd | ||
|
|
0adc5ab4e6 | ||
|
|
6079f634d6 | ||
|
|
eaf011e971 | ||
|
|
1c3b2597ba | ||
|
|
fd023f7473 | ||
|
|
5caa760f5e | ||
|
|
d63aae1d55 | ||
|
|
f8acfaeed7 | ||
|
|
0a9d1efe89 | ||
|
|
96a8690639 | ||
|
|
7868c9f365 | ||
|
|
225f2abb1f | ||
|
|
61806a7a92 | ||
|
|
61c42f9fbc | ||
|
|
86048f0ed9 | ||
|
|
039415cdd9 | ||
|
|
fb492da7e7 | ||
|
|
a12417fc13 | ||
|
|
8656e57eb9 | ||
|
|
650910dc9e | ||
|
|
321a0744d8 | ||
|
|
860ccd7e30 | ||
|
|
23799a30ed | ||
|
|
e558510a54 | ||
|
|
1d409cd852 | ||
|
|
302dcc098f | ||
|
|
3167f0104f | ||
|
|
0193949efe | ||
|
|
8dc186944e | ||
|
|
905265a56b | ||
|
|
4ef07521eb | ||
|
|
a9b9709911 | ||
|
|
df1412081b | ||
|
|
8e993c759a | ||
|
|
0c37ac378c | ||
|
|
d18bf8b3fd | ||
|
|
bef467febe | ||
|
|
a1a328c43e | ||
|
|
77e6e48ecd | ||
|
|
8734d5ee1a | ||
|
|
a3fea9106a | ||
|
|
4b215ba6ae | ||
|
|
4021de1d42 | ||
|
|
9afb287b9d | ||
|
|
c2710f52dd | ||
|
|
a3b646b3c9 | ||
|
|
732def66b0 | ||
|
|
ac3422f914 | ||
|
|
4d656fe4e1 | ||
|
|
94f6c3dba4 | ||
|
|
7d06ef7997 | ||
|
|
efee30e243 | ||
|
|
e8bff57b55 | ||
|
|
c6df8fa20f | ||
|
|
054ca5a6b3 | ||
|
|
b3b13f2ae1 | ||
|
|
0f108fd8ac | ||
|
|
0dd32d9722 | ||
|
|
31697e696d | ||
|
|
6441ae7e96 | ||
|
|
5694f6c39b | ||
|
|
ece79788a3 | ||
|
|
dc1aee51d9 | ||
|
|
cf378f9a25 | ||
|
|
2b916ac043 | ||
|
|
7c7b5e2d68 | ||
|
|
5173f2a31c | ||
|
|
97d3df3e59 | ||
|
|
7703162c95 | ||
|
|
38e13ef030 | ||
|
|
189ed10c36 | ||
|
|
75f5c64646 | ||
|
|
5d3b446988 | ||
|
|
ce01ec1f68 | ||
|
|
602ed830e9 | ||
|
|
a073a001c5 | ||
|
|
45e2b946c7 | ||
|
|
b41a0d59b6 | ||
|
|
f3813a279e | ||
|
|
420087ab14 | ||
|
|
26581d87ed | ||
|
|
730860b274 | ||
|
|
decbce4c54 | ||
|
|
14c22b4a45 | ||
|
|
e64c8ad0ca | ||
|
|
6304332ad5 | ||
|
|
783a2d2207 | ||
|
|
d581acdeda | ||
|
|
9cd03a7430 | ||
|
|
5aa6d39a98 | ||
|
|
4685610c19 | ||
|
|
f50442ac7a | ||
|
|
5c353cfc7b | ||
|
|
a484d5f960 | ||
|
|
4af6c274b8 | ||
|
|
7bf7e29a32 | ||
|
|
81bc0cc388 | ||
|
|
9d8647aa0f | ||
|
|
aa649d85a0 | ||
|
|
bcd815a6bb | ||
|
|
6e461bd24a | ||
|
|
ef41fa2e27 | ||
|
|
868ee6beb0 | ||
|
|
7c483da93a | ||
|
|
17a4ac620c | ||
|
|
95f45b1a74 | ||
|
|
2019122379 | ||
|
|
e504c12940 | ||
|
|
25d677d20c | ||
|
|
6bde46cb00 | ||
|
|
3ea81943f5 | ||
|
|
3dc4307aec | ||
|
|
5f37a43b91 | ||
|
|
9b55bcb127 | ||
|
|
9f6801a6c5 | ||
|
|
22a37b758c | ||
|
|
25aa110920 | ||
|
|
71508cc962 | ||
|
|
87bdc95a3d | ||
|
|
7b42c94a4d | ||
|
|
6dd8e08a84 | ||
|
|
60f413f9bc | ||
|
|
5108c28e22 | ||
|
|
875b4ee5ce | ||
|
|
9ab85ef92e | ||
|
|
89475bb0cf | ||
|
|
4f1e92160a | ||
|
|
fe9533c186 | ||
|
|
70e70eda5a | ||
|
|
cb7a444a56 | ||
|
|
53addd73c3 | ||
|
|
b6c174a324 | ||
|
|
5b75e866d0 | ||
|
|
d0fb384f00 | ||
|
|
c114b786e8 | ||
|
|
e96a7ada50 | ||
|
|
eec35a7c0c | ||
|
|
3b700886f2 | ||
|
|
a397492503 | ||
|
|
dae2aa0be1 | ||
|
|
d064c41057 | ||
|
|
393f6902b3 | ||
|
|
015304f4a3 | ||
|
|
c55672ce5e | ||
|
|
7add29cffe | ||
|
|
c1b8b9d115 | ||
|
|
8717ec76d4 | ||
|
|
e1b9f3e095 | ||
|
|
c40a79e601 | ||
|
|
b75495594a | ||
|
|
24b7740b80 | ||
|
|
aa40b107c9 | ||
|
|
d69fd304cc | ||
|
|
4a57d8c90c | ||
|
|
b8dbee28cc | ||
|
|
2bcc34a59b | ||
|
|
e282e08917 | ||
|
|
b0baa17836 | ||
|
|
ae26376a90 | ||
|
|
58f45a1b12 | ||
|
|
c1a1efa62a | ||
|
|
5b9e9d872d | ||
|
|
13e2fe14aa | ||
|
|
4bfa94dfef | ||
|
|
10852161c9 | ||
|
|
669589b76b | ||
|
|
c35cc1d9e4 | ||
|
|
3cf4fbaf3d | ||
|
|
7eb7b9fa62 | ||
|
|
ec3a65fa8d | ||
|
|
5906e18028 | ||
|
|
0871fa7cc4 | ||
|
|
cdd428a7d8 | ||
|
|
514ffc30da | ||
|
|
df44d21c82 | ||
|
|
dcbdc70919 | ||
|
|
84e50beb65 | ||
|
|
308dbb94b8 | ||
|
|
c642471c16 | ||
|
|
24cfc18383 | ||
|
|
f29b19aff2 | ||
|
|
855f4471e2 | ||
|
|
6ebad21cbe | ||
|
|
a222db8c69 | ||
|
|
66fe7c8a0d | ||
|
|
7623279efe | ||
|
|
570b66230c | ||
|
|
e5d9753368 | ||
|
|
6adaf44f65 | ||
|
|
66e89b71a2 | ||
|
|
32c5df952c | ||
|
|
4363c371c5 | ||
|
|
1b01d065ac | ||
|
|
d3ee49d6f3 | ||
|
|
ded92a8137 | ||
|
|
20614822df | ||
|
|
9740238ec9 | ||
|
|
a2313057c0 | ||
|
|
2b14f166de | ||
|
|
d8207358b0 | ||
|
|
c6ae0fc405 | ||
|
|
0128b962ab | ||
|
|
271de0108d | ||
|
|
c93feeb30f | ||
|
|
7f33c8b499 | ||
|
|
18c1735f7c | ||
|
|
2a71a8ff76 | ||
|
|
e73aae3686 | ||
|
|
3eef84d979 | ||
|
|
60358d01e9 | ||
|
|
05418b4fe5 | ||
|
|
2779f5312b | ||
|
|
e2380404a1 | ||
|
|
998d8f7370 | ||
|
|
5dc3a8e17f | ||
|
|
768f637bd0 | ||
|
|
ee0f46b78f | ||
|
|
84a14b08d5 | ||
|
|
fabfdf04f7 | ||
|
|
966093d8d0 | ||
|
|
4d47ff5ce7 | ||
|
|
8e32583089 | ||
|
|
6019c6d514 | ||
|
|
054754482c | ||
|
|
f7129fdb41 | ||
|
|
f7abd05981 | ||
|
|
514cf23aef | ||
|
|
95b509e0d2 | ||
|
|
83e135f101 | ||
|
|
30b86ea602 | ||
|
|
32a99144f6 | ||
|
|
3544d644ba | ||
|
|
a0ce37fb2c | ||
|
|
a24a1d72ad | ||
|
|
02c7f8e052 | ||
|
|
53b0e8d318 | ||
|
|
c777fbae8f | ||
|
|
98719a1bd5 | ||
|
|
72bcf20870 | ||
|
|
b47668421f | ||
|
|
2239fb253d | ||
|
|
7da4deb3f1 | ||
|
|
b461b005e9 | ||
|
|
75e3a26336 | ||
|
|
5b455cecfe | ||
|
|
faa2005538 | ||
|
|
6ea23b6731 | ||
|
|
db7cd273f0 | ||
|
|
12c80d7c2e | ||
|
|
649ed21d55 | ||
|
|
23cbf9294a | ||
|
|
03b7fcd7bb | ||
|
|
7b8f5e7805 | ||
|
|
798d73eb11 | ||
|
|
df13a83a91 | ||
|
|
61a3bd996d | ||
|
|
b8158affa7 | ||
|
|
ebb4f4af18 | ||
|
|
27a956448c | ||
|
|
0bbdae1d9d | ||
|
|
4c7056de42 | ||
|
|
ae4b86e333 | ||
|
|
8f1e5f2547 | ||
|
|
60518f532d | ||
|
|
cdd17eaae8 | ||
|
|
3293f6ddcd | ||
|
|
1ada760a64 | ||
|
|
d68e241724 | ||
|
|
be0226b624 | ||
|
|
acb720d2bf | ||
|
|
f6bda12a32 | ||
|
|
4b8b5c7329 | ||
|
|
3fa41dfe1a | ||
|
|
4f51393b72 | ||
|
|
9a12f2851f | ||
|
|
98fbc490a4 | ||
|
|
049c3b8c11 | ||
|
|
813f3cae87 | ||
|
|
77c1b50019 | ||
|
|
6bcc4eab24 | ||
|
|
692b0f0160 | ||
|
|
897a5efe7c | ||
|
|
98f4ff6163 | ||
|
|
ec4fe14457 | ||
|
|
75eb2bb534 | ||
|
|
18521ba7af | ||
|
|
a9a0f7b4e0 | ||
|
|
f5ccbb2700 | ||
|
|
0d1bbe967d | ||
|
|
ff4dc71ce4 | ||
|
|
84737f4cb2 | ||
|
|
38d666f223 | ||
|
|
5cfc4ff4a7 | ||
|
|
14a12cc7b6 | ||
|
|
44db32c31b | ||
|
|
12168ba4da | ||
|
|
72aca552e1 | ||
|
|
9ae30a7bbb | ||
|
|
e362c800a4 | ||
|
|
c7468656af | ||
|
|
598b0d8193 | ||
|
|
c27fad5c6e | ||
|
|
85826c633d | ||
|
|
9f7e898ecf | ||
|
|
74ecda8ea2 | ||
|
|
e61141f596 | ||
|
|
bc960c51db | ||
|
|
07066fe7a0 | ||
|
|
974177a074 | ||
|
|
0422893873 | ||
|
|
1d3029fa1e | ||
|
|
af66f72931 | ||
|
|
233bb23bbc | ||
|
|
e031758e4a | ||
|
|
122f4dd727 | ||
|
|
e0261cdfc1 | ||
|
|
6c1f7cc7f5 | ||
|
|
71afdf3ad2 | ||
|
|
ffa63d25a0 | ||
|
|
76c7ddfc52 | ||
|
|
39365c62c9 | ||
|
|
a4713e2d28 | ||
|
|
8bdfddab4a | ||
|
|
a4c0cee251 | ||
|
|
bb8383b065 | ||
|
|
da0f8f1cfd | ||
|
|
62b211bb60 | ||
|
|
3d1465606b | ||
|
|
1d8d726758 | ||
|
|
b0d6684d37 | ||
|
|
56eb0b792c | ||
|
|
9c15610f70 | ||
|
|
91fb323ce0 | ||
|
|
8da56ead81 | ||
|
|
7bbba2e39e | ||
|
|
08e9d15817 | ||
|
|
1b624c3ec7 | ||
|
|
7e2d947f5f | ||
|
|
1e833feb39 | ||
|
|
62cfb45789 | ||
|
|
16b8f591ed | ||
|
|
15361c661f | ||
|
|
cc3f7e8e01 | ||
|
|
d5e8458340 | ||
|
|
b133a8949d | ||
|
|
33131dcbfe | ||
|
|
2ae7c53376 | ||
|
|
4055e0094b | ||
|
|
ee71ba4dea | ||
|
|
b9c24a7b42 | ||
|
|
744bfec4a0 | ||
|
|
8bff438e5e | ||
|
|
7ecf7ba356 | ||
|
|
737bce9c94 | ||
|
|
ec191247a0 | ||
|
|
a9edd2951c | ||
|
|
015334020a | ||
|
|
a3c098a5c6 | ||
|
|
2fd165f5af | ||
|
|
db1151c773 | ||
|
|
02f78c9949 | ||
|
|
192633edb8 | ||
|
|
e5e8ff5783 | ||
|
|
c3ff18de4e | ||
|
|
3ed9abf658 | ||
|
|
552f97c919 | ||
|
|
65fbc1e6f8 | ||
|
|
cbe8528f91 | ||
|
|
5497d362dd | ||
|
|
6b32e07164 | ||
|
|
f7814853f9 | ||
|
|
58aacaad2d | ||
|
|
81672b183d | ||
|
|
76e214a8b9 | ||
|
|
acf02d52ff | ||
|
|
a7acee3eed | ||
|
|
d2645a27e9 | ||
|
|
7d39510450 | ||
|
|
f38ac87ac9 | ||
|
|
d8de0a8853 | ||
|
|
45d0348044 | ||
|
|
8c5af3e4a3 | ||
|
|
ece607e215 | ||
|
|
86a05babed | ||
|
|
1ec73915b8 | ||
|
|
f81e6147ae | ||
|
|
0a4c35f731 | ||
|
|
4f9227a186 | ||
|
|
fb8ea0e683 | ||
|
|
a6c70803ab | ||
|
|
9bd1c84e0f | ||
|
|
1f3fb52470 | ||
|
|
bb6e472939 | ||
|
|
93f33e2305 | ||
|
|
abdf85aa4f | ||
|
|
91686fdb81 | ||
|
|
d2a868f99f | ||
|
|
a143678fd8 | ||
|
|
9914459830 | ||
|
|
b7159d71a6 | ||
|
|
e0c7f36ca9 | ||
|
|
9120f94aa4 | ||
|
|
1900b40192 | ||
|
|
6cbdb3712e | ||
|
|
2a0c428ef9 | ||
|
|
a5d1f7dde4 | ||
|
|
95dd64c774 | ||
|
|
81fa009bf4 | ||
|
|
05ff90e3c1 | ||
|
|
aaffc19232 | ||
|
|
479ac71050 | ||
|
|
4f6b389a37 | ||
|
|
45f5e27526 | ||
|
|
dd8ba97989 | ||
|
|
ab22efd8b1 | ||
|
|
54aee345fc | ||
|
|
8639b373d2 | ||
|
|
5fc69444f6 | ||
|
|
2fe9d678fc | ||
|
|
58b4f8b6bc | ||
|
|
bf667a602d | ||
|
|
9a8b3870ba | ||
|
|
24c23a1d06 | ||
|
|
a548a61e86 | ||
|
|
bc28606d45 | ||
|
|
7f0eb9c51d | ||
|
|
690aa662a8 | ||
|
|
8bd0b5db15 | ||
|
|
b4fd2cc210 | ||
|
|
1a021052fe | ||
|
|
cf738cdb90 | ||
|
|
7a2af23764 | ||
|
|
daaa1962bd | ||
|
|
df878ba1f7 | ||
|
|
1e01868d92 | ||
|
|
9d04ae7edd | ||
|
|
16df7d0b7a | ||
|
|
cd1d2ff447 | ||
|
|
faf6a23d91 | ||
|
|
2f4e81f3b0 | ||
|
|
4182f8fbfb | ||
|
|
c3a2ffde67 | ||
|
|
6502ba6ad6 | ||
|
|
f6c0520f26 | ||
|
|
c912bcab6e | ||
|
|
63c0b445d7 | ||
|
|
653ff61662 | ||
|
|
a7cd0a7807 | ||
|
|
7271cf151b | ||
|
|
82d0e125b2 | ||
|
|
a4c24a1979 | ||
|
|
cff69893a3 | ||
|
|
047237c911 | ||
|
|
449cda1b00 | ||
|
|
a6e53a4858 | ||
|
|
4993f1b1d7 | ||
|
|
199f6b48b1 | ||
|
|
4636261435 | ||
|
|
4e8fdfdf04 | ||
|
|
d00926e089 | ||
|
|
bb15aab50e | ||
|
|
6a4718b712 | ||
|
|
d5adf9b72b | ||
|
|
7365890298 | ||
|
|
0862f97972 | ||
|
|
45439fd742 | ||
|
|
0184e2a5b8 | ||
|
|
50c229f905 | ||
|
|
69b845c0f7 | ||
|
|
caafd9306e | ||
|
|
0d22776b20 | ||
|
|
1803d25c3a | ||
|
|
d56c576e9a | ||
|
|
fbc6d6330f | ||
|
|
fb0cdabe25 | ||
|
|
0474a2a3cb | ||
|
|
180a7e7b80 | ||
|
|
be94d2689e | ||
|
|
b225c7bb4c | ||
|
|
1d44288514 | ||
|
|
af2d2a2c14 | ||
|
|
59d77dd457 | ||
|
|
23d66187a8 | ||
|
|
4476e58080 | ||
|
|
58de6aa10a | ||
|
|
cdd3f3fd4f | ||
|
|
21d6583b5e | ||
|
|
bb46a6ad0f | ||
|
|
f8afaefba0 | ||
|
|
1e38c42646 | ||
|
|
1241ee7041 | ||
|
|
2dc384d641 | ||
|
|
e4dbb49d2f | ||
|
|
02d9d7b053 | ||
|
|
7fc0ea5478 | ||
|
|
8be1c2ab0d | ||
|
|
ff9386dccc | ||
|
|
bba9081ca1 | ||
|
|
65cd7b4787 | ||
|
|
a78239b2aa | ||
|
|
2ac0c4686a | ||
|
|
5233f7584d | ||
|
|
8858dba25e | ||
|
|
9254180cab | ||
|
|
94f9aa298c | ||
|
|
ee9fcaba20 | ||
|
|
43f2d6020e | ||
|
|
6250b1274b | ||
|
|
7f04d12047 | ||
|
|
e88ddab241 | ||
|
|
06f70e11ee | ||
|
|
61cec66ba8 | ||
|
|
5e2fadce15 | ||
|
|
882e75c954 | ||
|
|
1a190090ab | ||
|
|
fd381567f4 | ||
|
|
1195cf4597 | ||
|
|
99285cb080 | ||
|
|
d75c66dc7e | ||
|
|
e196058f6f | ||
|
|
5e71ab6ae6 | ||
|
|
3f02bd7ff6 | ||
|
|
338797a527 | ||
|
|
2c93f79429 | ||
|
|
5f16cf507a | ||
|
|
b32a27c115 | ||
|
|
c16e625a08 | ||
|
|
1cdb4d1a29 | ||
|
|
0dfbe1a179 | ||
|
|
1bdbf47ffa | ||
|
|
16f84b53da | ||
|
|
57b553ac5a | ||
|
|
11a635796d | ||
|
|
8343c4fb36 | ||
|
|
3e3e43de52 | ||
|
|
175fb45c19 | ||
|
|
219169dbd9 | ||
|
|
b99d0b45bf | ||
|
|
f06058390c | ||
|
|
958ccbde3e | ||
|
|
43d1c2456f | ||
|
|
c46937da04 | ||
|
|
cf455bc4fe | ||
|
|
ecf142e451 | ||
|
|
fa9e4483c8 | ||
|
|
e62c32c8c3 | ||
|
|
2bfde7d659 | ||
|
|
db06feed2e | ||
|
|
4e6b756bd2 | ||
|
|
b833d09d28 | ||
|
|
81758b24a8 | ||
|
|
25ad5ce42c | ||
|
|
285395d523 | ||
|
|
b47df31834 | ||
|
|
fd05c8235d | ||
|
|
98d1f165b7 | ||
|
|
332d8f161b | ||
|
|
5088755e5e | ||
|
|
e72c682450 | ||
|
|
be5f31ce66 | ||
|
|
67db003813 | ||
|
|
e5a06cc73d | ||
|
|
20d0998cdd | ||
|
|
83f16cff8d | ||
|
|
96d0322498 | ||
|
|
b2003e7a38 | ||
|
|
3303797bfb | ||
|
|
d5dd7761c6 | ||
|
|
5f5f37b0b2 | ||
|
|
c30b86ae22 | ||
|
|
1598156752 | ||
|
|
e2b6f37168 | ||
|
|
13bdcb694b | ||
|
|
89ebd0eefe | ||
|
|
b4f6537f80 | ||
|
|
7618f661ea | ||
|
|
3c3628c0ce | ||
|
|
4b4bb09a05 | ||
|
|
11b0a1fb29 | ||
|
|
6ea77844d6 | ||
|
|
2633573012 | ||
|
|
976c9998d6 | ||
|
|
48c3e3834c | ||
|
|
0d12fb11a4 | ||
|
|
03b9c47af4 | ||
|
|
cdbff1777e | ||
|
|
59e6a6fa2a | ||
|
|
8c0bb9d453 | ||
|
|
de809d7e0a | ||
|
|
4065b45ae8 | ||
|
|
09c77d57b8 | ||
|
|
c231d1031f | ||
|
|
e597f9791c | ||
|
|
e3a595d8fe | ||
|
|
7e83137aa8 | ||
|
|
f9e1cd183a | ||
|
|
d9f8bbec0c | ||
|
|
d5eeb860e3 | ||
|
|
ef30c12291 | ||
|
|
fd3259a3f5 | ||
|
|
fdb59bcf4e | ||
|
|
d0298a99da | ||
|
|
4cfa0469fd | ||
|
|
45f437eea6 | ||
|
|
a457b38968 | ||
|
|
c5c4caa2a4 | ||
|
|
5a2bc03910 | ||
|
|
60371aadad | ||
|
|
ce1b97ad97 | ||
|
|
52600ec7bd | ||
|
|
278a94d59b | ||
|
|
9d76759e04 | ||
|
|
a8eb24b49c | ||
|
|
da26b7f130 | ||
|
|
cdcba9ca73 | ||
|
|
454be9b269 | ||
|
|
619e14aa8e | ||
|
|
d6f8b00bf8 | ||
|
|
a9789b5125 | ||
|
|
150fadfe8b | ||
|
|
6a2fb63bfc | ||
|
|
64dfa90c3a | ||
|
|
2474bf2833 | ||
|
|
671f284004 | ||
|
|
89025ad004 | ||
|
|
3798673821 | ||
|
|
8119c470d3 | ||
|
|
aab65b81f6 | ||
|
|
9d9e3a46c4 | ||
|
|
0b391f49a2 | ||
|
|
6469e82384 | ||
|
|
dae79c7643 | ||
|
|
d237dee4c9 | ||
|
|
cd13f7726e | ||
|
|
bff1ac14a1 | ||
|
|
49c9002c04 | ||
|
|
c3b2fce0a2 | ||
|
|
198fd34ec4 | ||
|
|
62af89f85d | ||
|
|
1a26778823 | ||
|
|
b9a293f8c8 | ||
|
|
695e022991 | ||
|
|
0e92ef3d1b | ||
|
|
12bc6a56a3 | ||
|
|
cdcfcd9380 | ||
|
|
130c5d98a6 | ||
|
|
cb7b71dcc5 | ||
|
|
8108dd5c23 | ||
|
|
f0d39b4a4a | ||
|
|
89fd0c06e1 | ||
|
|
17038b2a07 | ||
|
|
4dde21f0ab | ||
|
|
7d1d6bffe3 | ||
|
|
7283209d1a | ||
|
|
64f0ab4051 | ||
|
|
5f147fe6b9 | ||
|
|
0d11b4fc93 | ||
|
|
d34b4f32e1 | ||
|
|
d941d798e9 | ||
|
|
b7f70bc4c0 | ||
|
|
8431ac9656 | ||
|
|
bc2a448512 | ||
|
|
bb60e7b584 | ||
|
|
7a41e12063 | ||
|
|
c7b534a08c | ||
|
|
ccc86d3594 | ||
|
|
6fe82c61bd | ||
|
|
8e95c1d9ed | ||
|
|
cb55a46400 | ||
|
|
37947ca4ee | ||
|
|
d8e785bb4b | ||
|
|
281837847e | ||
|
|
c7551162d6 | ||
|
|
2cffa33bcc | ||
|
|
2a2be687f3 | ||
|
|
303d4037e8 | ||
|
|
5c8e768ef5 | ||
|
|
aad33dfe5c | ||
|
|
ef26dba275 | ||
|
|
d12bba0109 | ||
|
|
c85c2804a0 | ||
|
|
2ff40109ac | ||
|
|
46c16e18fe | ||
|
|
f00f9c2a2f | ||
|
|
21cd036232 | ||
|
|
f0adb177f0 | ||
|
|
d2d9fbbd7d | ||
|
|
2026ef36f3 | ||
|
|
51e97a8c14 | ||
|
|
c7a4bf5c11 | ||
|
|
ec983bfc3d | ||
|
|
dd9ab7927d | ||
|
|
0818e4c29a | ||
|
|
41e640d668 | ||
|
|
a1f9856b39 | ||
|
|
74ed7cbe0a | ||
|
|
36704b5471 | ||
|
|
6e44a27a63 | ||
|
|
dce0eef1ca | ||
|
|
e3c3bb37d0 | ||
|
|
b92fb68116 | ||
|
|
3f7f797168 | ||
|
|
c433e76de2 | ||
|
|
f4efc3cad0 | ||
|
|
a76baa4b47 | ||
|
|
90ce9e9d9f | ||
|
|
df5de3bd9f | ||
|
|
a18d09db75 | ||
|
|
3333a05a74 | ||
|
|
4695dfc0a2 | ||
|
|
7233ddbe35 | ||
|
|
1acfed4add | ||
|
|
6defec8f68 | ||
|
|
2833caf3d6 | ||
|
|
20d5af9b2d | ||
|
|
0ec96a048e | ||
|
|
c957f1ec5e | ||
|
|
85c51ebb59 | ||
|
|
f9805b2113 | ||
|
|
70cb613614 | ||
|
|
bd5cc69804 | ||
|
|
f311fa36dc | ||
|
|
30242197fa | ||
|
|
8f7db549ae | ||
|
|
5226d7bcd0 | ||
|
|
9545121ec9 | ||
|
|
3abb07f1e2 | ||
|
|
9d8aa9d6f0 | ||
|
|
212a3cc113 | ||
|
|
cc0b8554b1 | ||
|
|
0b4c86e3be | ||
|
|
8e79403c21 | ||
|
|
b8c8546118 | ||
|
|
a7270f00cb | ||
|
|
2dbbda51c6 | ||
|
|
2410796869 | ||
|
|
090b9082eb | ||
|
|
8fc1d72cd5 | ||
|
|
9a807de659 | ||
|
|
fc6c5a92e6 | ||
|
|
e74ee82a76 | ||
|
|
b0d03be7a4 | ||
|
|
4ea6a2f04e | ||
|
|
d8ff1540df | ||
|
|
cdc2af7b43 | ||
|
|
7a2ffc0a4c | ||
|
|
b1e7b6d26e | ||
|
|
512b0edb02 | ||
|
|
4ba6d0864e | ||
|
|
73e0916968 | ||
|
|
bf3f45e769 | ||
|
|
fd13834c0d | ||
|
|
4a5dcb0209 | ||
|
|
1ac7741c63 | ||
|
|
f684967c3b | ||
|
|
655f515aa7 | ||
|
|
8f763ee911 | ||
|
|
564948c2f2 | ||
|
|
1c61b72f35 | ||
|
|
2673e7a2c9 | ||
|
|
021d860ca4 | ||
|
|
b28d7bbcbf | ||
|
|
e2e8be72fa | ||
|
|
d4af8bba1b | ||
|
|
02a1c5f753 | ||
|
|
b89b143fe7 | ||
|
|
3a727011ad | ||
|
|
d93cc86f95 | ||
|
|
a390bce5f2 | ||
|
|
c2bae14994 | ||
|
|
690966d847 | ||
|
|
d28ca959f0 | ||
|
|
b232db4e17 | ||
|
|
dfb14151bd | ||
|
|
a018ebd1ca | ||
|
|
7c046a8a1d | ||
|
|
9d48b8baf9 | ||
|
|
ff2553f4db | ||
|
|
ae1293e73e | ||
|
|
a2c2c35e85 | ||
|
|
07164e40f3 | ||
|
|
f39dbdbaf0 | ||
|
|
3a002f6e7b | ||
|
|
ee20388239 | ||
|
|
fedd9dcf35 | ||
|
|
29a7de7699 | ||
|
|
a31659ed03 | ||
|
|
ccbfa9b58a | ||
|
|
e813391dcf | ||
|
|
5684a4579d | ||
|
|
824911dc50 | ||
|
|
5e189c2fca | ||
|
|
809ab4cf29 | ||
|
|
6400e92d19 | ||
|
|
41b3392789 | ||
|
|
e41ab3a505 | ||
|
|
800aabed83 | ||
|
|
9d8398206e | ||
|
|
9815796891 | ||
|
|
ab4ca44f98 | ||
|
|
ef8378a990 | ||
|
|
14e5d91c9a | ||
|
|
f863adb741 | ||
|
|
86cbebd10f | ||
|
|
0e57186bf1 | ||
|
|
e30d838395 | ||
|
|
0863695012 | ||
|
|
c38f21bb39 | ||
|
|
fcca8d9ce8 | ||
|
|
f8ef2de91d | ||
|
|
babd32cfc7 | ||
|
|
039abeff69 | ||
|
|
429ea42357 | ||
|
|
56e2ebec4b | ||
|
|
96bf9415aa | ||
|
|
bdd44154cc | ||
|
|
41f908b803 | ||
|
|
3c1ac6aef2 | ||
|
|
038eff5ca0 | ||
|
|
3a117c2d2b |
@@ -4,3 +4,5 @@
|
|||||||
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
|
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
|
||||||
# interrupted release build cannot commit a twenty-megabyte tar by accident.
|
# interrupted release build cannot commit a twenty-megabyte tar by accident.
|
||||||
/internal/image/builder.tar.placeholder
|
/internal/image/builder.tar.placeholder
|
||||||
|
/cmd/mesh-host/mesh-host
|
||||||
|
/cmd/mesh-bootstrap/mesh-bootstrap
|
||||||
|
|||||||
@@ -4,11 +4,16 @@ Tier 0 of the Novox Mesh. The one thing ever installed by hand, and the only thi
|
|||||||
a machine.
|
a machine.
|
||||||
|
|
||||||
```
|
```
|
||||||
scp mesh-host root@machine:/usr/local/bin/
|
scp nox-mesh-host root@machine:/usr/bin/
|
||||||
mesh-host profile
|
nox-mesh-host profile
|
||||||
```
|
```
|
||||||
|
|
||||||
That is the whole installation. One statically linked binary, nothing else present, no runtime
|
That is the whole installation. **It is a daemon**: the launcher runs it, and the operator's command is
|
||||||
|
`nox` (`mesh-cli`), on any node, from their own account (`novox/hq` ADR 0272). The subcommands below are
|
||||||
|
for installing a machine, joining it, and rescuing one no mesh can reach, run as root at the machine; a
|
||||||
|
line the mesh addresses to the operator names a `nox` line, never this binary. Once a delivered engine
|
||||||
|
proves itself, it renews the one placed by hand here to its own build, so the command on the machine is
|
||||||
|
the engine that runs it (`novox/hq` issue 356). One statically linked binary, nothing else present, no runtime
|
||||||
to install first ([`novox/hq` ADR 0005](https://git.novox.be/novox/hq)).
|
to install first ([`novox/hq` ADR 0005](https://git.novox.be/novox/hq)).
|
||||||
|
|
||||||
## What it is for
|
## What it is for
|
||||||
@@ -23,7 +28,7 @@ the host never queries the mesh database. It receives declarations and applies t
|
|||||||
## Joining a mesh
|
## Joining a mesh
|
||||||
|
|
||||||
```
|
```
|
||||||
mesh-host enrol --token <token> --name <what this machine is called>
|
nox-mesh-host enrol --token <token> --name <what this machine is called>
|
||||||
```
|
```
|
||||||
|
|
||||||
**The node generates its own identity** — an Ed25519 keypair whose private half never leaves the
|
**The node generates its own identity** — an Ed25519 keypair whose private half never leaves the
|
||||||
@@ -52,20 +57,22 @@ and then stops, having saved nothing — so it can be run again unchanged.
|
|||||||
connects to nothing and listens on nothing — what it applies comes from a file.
|
connects to nothing and listens on nothing — what it applies comes from a file.
|
||||||
|
|
||||||
```
|
```
|
||||||
mesh-host profile what this machine can be asked to do
|
nox-mesh-host profile what this machine can be asked to do
|
||||||
mesh-host inventory what this machine is, and what it holds
|
nox-mesh-host inventory what this machine is, and what it holds
|
||||||
mesh-host apply FILE make this machine match a declaration from a file
|
nox-mesh-host apply FILE make this machine match a declaration from a file
|
||||||
mesh-host reconcile make this machine match what the mesh last told it — or, before
|
nox-mesh-host reconcile make this machine match what the mesh last told it — or, before
|
||||||
any mesh has, the bundle this host carries
|
any mesh has, the bundle this host carries
|
||||||
mesh-host bundle show what this host carries
|
nox-mesh-host bundle show what this host carries
|
||||||
mesh-host owned what this host has applied and still owns
|
nox-mesh-host owned what this host has applied and still owns
|
||||||
|
nox-mesh-host hand-over DIR hand a directory used as found to the mesh, on a machine no mesh
|
||||||
|
can reach; the operator's line is `nox node hand-over <node> DIR`
|
||||||
--json machine-readable
|
--json machine-readable
|
||||||
--state where this node keeps what it knows
|
--state where this node keeps what it knows
|
||||||
--dry-run say what applying would change, and change nothing
|
--dry-run say what applying would change, and change nothing
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```
|
||||||
$ mesh-host profile
|
$ nox-mesh-host profile
|
||||||
linux/amd64
|
linux/amd64
|
||||||
|
|
||||||
yes container-runtime 29.7.2
|
yes container-runtime 29.7.2
|
||||||
@@ -95,6 +102,19 @@ what it did not understand would apply most of a declaration and report success.
|
|||||||
]}
|
]}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**A directory or archive says how it treats files it did not write** (issue 1024,
|
||||||
|
[`novox/hq` ADR 0422](https://git.novox.be/novox/hq)). `unknown` is `move-out`, `module` or `program`;
|
||||||
|
absent, nothing about unknown paths is done. `knows` lists paths inside it, relative to its `path`, that
|
||||||
|
are not declared but belong there; a trailing `/` names a folder and everything in it. `knows` without
|
||||||
|
`unknown` is refused. For `move-out` the node-engine, after every apply and so on every reconcile, moves
|
||||||
|
each path inside the directory that nothing declared, no archive unpacked and `knows` does not name,
|
||||||
|
whole, into `<home>/.local/state/mesh-moved/<module>/<UTC time>/` beside a `<UTC time>.json` list. The
|
||||||
|
list names each move before it is made. Nothing is deleted. A path younger than a minute waits for the
|
||||||
|
next look; a socket, pipe or device is left and said. It is only for a directory under an account's
|
||||||
|
home on one file system with the kept folder, and otherwise the resource fails and nothing moves.
|
||||||
|
`module` and `program` are read by the module and the program that owns the path, and the node-engine
|
||||||
|
does nothing for them.
|
||||||
|
|
||||||
**It converges rather than executes.** Applying twice changes nothing the second time; applying
|
**It converges rather than executes.** Applying twice changes nothing the second time; applying
|
||||||
to a drifted machine returns it. A mode is *maintained*, not merely set — a permission applied
|
to a drifted machine returns it. A mode is *maintained*, not merely set — a permission applied
|
||||||
at creation is not a permission held.
|
at creation is not a permission held.
|
||||||
@@ -115,7 +135,7 @@ A host built for a machine carries its declaration **inside the binary**:
|
|||||||
make host BUNDLE=path/to/foundation.lock
|
make host BUNDLE=path/to/foundation.lock
|
||||||
```
|
```
|
||||||
|
|
||||||
`mesh-host reconcile` then applies it, on a machine the mesh has told nothing yet. That is the
|
`nox-mesh-host reconcile` then applies it, on a machine the mesh has told nothing yet. That is the
|
||||||
first node's path — no mesh present, nothing fetched, nothing else copied onto the machine. Once
|
first node's path — no mesh present, nothing fetched, nothing else copied onto the machine. Once
|
||||||
the mesh has spoken, `reconcile` holds the machine to what it last said and never to the bundle,
|
the mesh has spoken, `reconcile` holds the machine to what it last said and never to the bundle,
|
||||||
which genesis consumed; a bundle or a file is refused when it says the other mode than the node
|
which genesis consumed; a bundle or a file is refused when it says the other mode than the node
|
||||||
@@ -217,7 +237,7 @@ without its neighbour checked out is a repository nobody can build.
|
|||||||
|
|
||||||
```
|
```
|
||||||
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
||||||
mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
|
mesh-sdk: MESH_EMITTED=/tmp/d.json go test ./declaration/ -v (run in mesh-sdk's go/ directory)
|
||||||
```
|
```
|
||||||
|
|
||||||
**What this node says when it joins, read by the mesh:**
|
**What this node says when it joins, read by the mesh:**
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Release 57: config and provisioning
|
||||||
|
|
||||||
|
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||||
|
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||||
|
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||||
|
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||||
@@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
|||||||
|
|
||||||
--tools-source the repository the shared base is built from
|
--tools-source the repository the shared base is built from
|
||||||
--tools-ref what of it to build (default main)
|
--tools-ref what of it to build (default main)
|
||||||
|
--tools-go-source the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)
|
||||||
|
--tools-go-ref what of it to build (default main)
|
||||||
--catalog-source the catalogue REPOSITORY, for building its modules;
|
--catalog-source the catalogue REPOSITORY, for building its modules;
|
||||||
--catalog is the checkout that says what they are
|
--catalog is the checkout that says what they are
|
||||||
--catalog-ref what of it to build (default main)
|
--catalog-ref what of it to build (default main)
|
||||||
@@ -130,7 +132,7 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
|||||||
|
|
||||||
The foundation's ports are this machine's, each checked free before anything is
|
The foundation's ports are this machine's, each checked free before anything is
|
||||||
raised and kept as the node's setting for the module that binds it:
|
raised and kept as the node's setting for the module that binds it:
|
||||||
--store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672
|
--store-port 5432 --bus-port 5671
|
||||||
--registry-port 5000 (follows --registry, and must agree with it)
|
--registry-port 5000 (follows --registry, and must agree with it)
|
||||||
--packages-port 3000 --hub-port 51820/udp
|
--packages-port 3000 --hub-port 51820/udp
|
||||||
--overlay-range the private network's range (default 10.42.0.0/16); refused
|
--overlay-range the private network's range (default 10.42.0.0/16); refused
|
||||||
@@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
|||||||
"the repository the shared base is built from")
|
"the repository the shared base is built from")
|
||||||
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
|
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
|
||||||
"what of it to build (default main)")
|
"what of it to build (default main)")
|
||||||
|
set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository,
|
||||||
|
"the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)")
|
||||||
|
set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref,
|
||||||
|
"what of it to build (default main)")
|
||||||
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
|
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
|
||||||
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
|
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
|
||||||
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
|
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
|
||||||
@@ -325,9 +331,7 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
|||||||
into *int
|
into *int
|
||||||
}{
|
}{
|
||||||
{"store-port", "the store", &opts.Ports.Store},
|
{"store-port", "the store", &opts.Ports.Store},
|
||||||
{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
|
{"bus-port", "the bus", &opts.Ports.Bus},
|
||||||
{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
|
|
||||||
{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
|
|
||||||
{"registry-port", "the registry", &opts.Ports.Registry},
|
{"registry-port", "the registry", &opts.Ports.Registry},
|
||||||
{"packages-port", "the package registry", &opts.Ports.Packages},
|
{"packages-port", "the package registry", &opts.Ports.Packages},
|
||||||
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
|
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
|
||||||
|
|||||||
@@ -0,0 +1,212 @@
|
|||||||
|
// mesh-compose is the node-engine's composer command (novox/hq ADR 0482 clause 4, ADR 0593 §7, issue 1621): it reads a
|
||||||
|
// composer.Request on standard input, reads every manifest, composes the machines named in each case and writes one
|
||||||
|
// composer.Verdict on standard output. Built at a commit of the node-engine's repository, it answers what that
|
||||||
|
// commit's composition answers, so a group that changes the node-engine's composition is judged by its own next code.
|
||||||
|
//
|
||||||
|
// mesh-compose < request.json > verdict.json
|
||||||
|
// mesh-compose -request request.json > verdict.json
|
||||||
|
// mesh-compose -catalogue <dir> [-node <name>]... [-depth resolve|compose] [-declarations] [< request.json]
|
||||||
|
//
|
||||||
|
// -catalogue reads <dir>/modules/*/module.json and a module.json at <dir>'s root as the first case's manifests (a
|
||||||
|
// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
|
||||||
|
// compose; -declarations answers each machine's declared paths with their owners and modes.
|
||||||
|
//
|
||||||
|
// Run as a pipeline step it reads its input, a composer.Request at depth compose, from the file named by the environment
|
||||||
|
// variable MESH_RUN_INPUT and answers the run's output as its composer.Verdict on the last line of standard output
|
||||||
|
// (design 47 G A-167).
|
||||||
|
//
|
||||||
|
// It opens no network connection, bus, store or file beyond its standard input, the -request file, the file
|
||||||
|
// MESH_RUN_INPUT names and the -catalogue directory, and reads no other environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
|
||||||
|
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/composer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodes []string
|
||||||
|
|
||||||
|
func (n *nodes) String() string { return strings.Join(*n, ",") }
|
||||||
|
func (n *nodes) Set(v string) error { *n = append(*n, v); return nil }
|
||||||
|
|
||||||
|
// run is the command: its arguments, standard input, output and error; it answers the exit code.
|
||||||
|
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||||
|
fs := flag.NewFlagSet("mesh-compose", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
catalogueDir := fs.String("catalogue", "", "a catalogue checkout: <dir>/modules/*/module.json are the first case's manifests")
|
||||||
|
var machines nodes
|
||||||
|
fs.Var(&machines, "node", "a machine to compose (repeatable)")
|
||||||
|
depth := fs.String("depth", "", "resolve or compose (default: the request's, else compose)")
|
||||||
|
declarations := fs.Bool("declarations", false, "answer each composed machine's declared paths, owners and modes")
|
||||||
|
requestFile := fs.String("request", "", "read the request from this file instead of standard input (a run step that hands no standard input)")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
fmt.Fprintf(stderr, "mesh-compose: unexpected arguments %q\n", fs.Args())
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
var raw []byte
|
||||||
|
var err error
|
||||||
|
// A pipeline step hands its input as a file named by MESH_RUN_INPUT (design 47 G A-167); -request, when given, wins.
|
||||||
|
runInput := os.Getenv("MESH_RUN_INPUT")
|
||||||
|
if *requestFile == "" && runInput != "" {
|
||||||
|
*requestFile = runInput
|
||||||
|
}
|
||||||
|
if *requestFile != "" {
|
||||||
|
raw, err = os.ReadFile(*requestFile)
|
||||||
|
} else {
|
||||||
|
raw, err = io.ReadAll(stdin)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: the request cannot be read:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
var req composer.Request
|
||||||
|
if strings.TrimSpace(string(raw)) == "" {
|
||||||
|
if *catalogueDir == "" {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: no request on standard input and no -catalogue")
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
req = composer.Request{V: composer.Version}
|
||||||
|
} else if req, err = composer.Decode(raw); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if *catalogueDir != "" {
|
||||||
|
read, err := readCatalogue(*catalogueDir)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if len(req.Cases) == 0 {
|
||||||
|
req.Cases = []composer.Case{{Name: "catalogue", Manifests: map[string]composer.Manifest{}}}
|
||||||
|
}
|
||||||
|
if req.Cases[0].Manifests == nil {
|
||||||
|
req.Cases[0].Manifests = map[string]composer.Manifest{}
|
||||||
|
}
|
||||||
|
for name, m := range read {
|
||||||
|
if _, given := req.Cases[0].Manifests[name]; !given {
|
||||||
|
req.Cases[0].Manifests[name] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No list of predefined keys is read: the composer judges no manifest (novox/hq ADR 0578 §5).
|
||||||
|
}
|
||||||
|
req.Machines = append(req.Machines, machines...)
|
||||||
|
if *depth != "" {
|
||||||
|
req.Depth = *depth
|
||||||
|
}
|
||||||
|
if *declarations {
|
||||||
|
req.Declarations = true
|
||||||
|
}
|
||||||
|
v, err := runWithSeats(req)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
enc := json.NewEncoder(stdout)
|
||||||
|
if err := enc.Encode(v); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "mesh-compose: the verdict cannot be written:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoSeatList is what a machine reads, not judged, when the request carries no seat list (novox/hq ADR 0636 §1).
|
||||||
|
const NoSeatList = "the composer was handed no seat list, so it composes nothing (a reader of the seat list acts on nothing " +
|
||||||
|
"new until it has read the list, novox/hq ADR 0636 §1): the asker gives the seat definitions it judges against in the request's seats"
|
||||||
|
|
||||||
|
// runWithSeats composes req against the seat list the request carries (Request.Seats: the definitions the asker's seat list
|
||||||
|
// reader holds, the members' heads' laid over them in a check, ADR 0636 §4), over the foundation's bootstrapped ones
|
||||||
|
// (seatlist.Bootstrapped), as the node-engine composes against the list it read from the bus (ADR 0635 §1). The command
|
||||||
|
// compiles no seat list: without one in the request it reads every manifest and judges no machine, each said not judged
|
||||||
|
// with NoSeatList (ADR 0593 §9), never a refusal or a pass made without the seats' paths, deliveries and holders.
|
||||||
|
func runWithSeats(req composer.Request) (composer.Verdict, error) {
|
||||||
|
if len(req.Seats) == 0 {
|
||||||
|
machines := req.Machines
|
||||||
|
req.Machines = nil
|
||||||
|
v, err := composer.Run(req)
|
||||||
|
if err != nil || len(machines) == 0 {
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
for i := range v.Cases {
|
||||||
|
v.Cases[i].Machines = map[string]composer.Machine{}
|
||||||
|
for _, node := range machines {
|
||||||
|
// Blamed is always written, blaming none here (design 47 G §721, A-469h).
|
||||||
|
v.Cases[i].Machines[node] = composer.Machine{NotJudged: NoSeatList, Blamed: []struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Problem string `json:"problem"`
|
||||||
|
}{}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
defs := map[string]seatdef.Definition{}
|
||||||
|
for _, d := range seatlist.Bootstrapped() {
|
||||||
|
defs[d.Seat] = d
|
||||||
|
}
|
||||||
|
for _, raw := range req.Seats {
|
||||||
|
d, err := seatdef.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return composer.Verdict{}, fmt.Errorf("a seat definition of the request: %w", err)
|
||||||
|
}
|
||||||
|
defs[d.Seat] = d
|
||||||
|
}
|
||||||
|
catalogue.UseSeats(catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))))
|
||||||
|
return composer.Run(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readCatalogue reads every module.json of a catalogue checkout by its module's name: the root's and each of
|
||||||
|
// modules/*/. A manifest whose name cannot be read is kept under its directory's name, so the reader's refusal is said.
|
||||||
|
func readCatalogue(dir string) (map[string]composer.Manifest, error) {
|
||||||
|
paths, err := filepath.Glob(filepath.Join(dir, "modules", "*", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "module.json")); err == nil {
|
||||||
|
paths = append(paths, filepath.Join(dir, "module.json"))
|
||||||
|
}
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return nil, fmt.Errorf("the catalogue %s holds no module.json", dir)
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
out := map[string]composer.Manifest{}
|
||||||
|
for _, p := range paths {
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var probe struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
name := filepath.Base(filepath.Dir(p))
|
||||||
|
if json.Unmarshal(raw, &probe) == nil && probe.Module != "" {
|
||||||
|
name = probe.Module
|
||||||
|
}
|
||||||
|
if _, twice := out[name]; twice {
|
||||||
|
return nil, errors.New("the catalogue " + dir + " defines " + name + " twice")
|
||||||
|
}
|
||||||
|
out[name] = composer.Manifest{Raw: raw}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,509 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets/bucketstest"
|
||||||
|
"go/build"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||||
|
"github.com/novox/mesh-host/internal/beside"
|
||||||
|
"github.com/novox/mesh-host/internal/compose"
|
||||||
|
"github.com/novox/mesh-host/internal/composer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// captured is every manifest of the catalogue captured in testdata/beside (internal/beside), raw, by module name.
|
||||||
|
func captured(t *testing.T) map[string]composer.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
read, err := readCatalogue(filepath.Dir(beside.Catalogue(t)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(read) < 100 {
|
||||||
|
t.Fatalf("the captured catalogue holds %d manifests, fewer than the catalogue's", len(read))
|
||||||
|
}
|
||||||
|
return read
|
||||||
|
}
|
||||||
|
|
||||||
|
// meshOf is a mesh of three machines over the captured catalogue, each assigned real modules, the input the assigner
|
||||||
|
// composes with (nodes and their assignments, holdings from the claims): anchor runs the bus and the store, laptop a
|
||||||
|
// desktop's modules, ace a module whose requirement is provided elsewhere.
|
||||||
|
func meshOf() compose.Input {
|
||||||
|
return compose.Input{Nodes: []compose.Node{
|
||||||
|
{Name: "anchor", Assigned: []string{"nats", "postgres"}},
|
||||||
|
{Name: "laptop", Assigned: []string{"sudo", "zsh"}},
|
||||||
|
{Name: "ace", Assigned: []string{"mesh-reads", "nats"}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func request(t *testing.T, depth string) composer.Request {
|
||||||
|
t.Helper()
|
||||||
|
r := composer.Request{V: composer.Version, Depth: depth, HoldingsFromClaims: true,
|
||||||
|
Machines: []string{"anchor", "laptop", "ace"}, Seats: seats(t),
|
||||||
|
Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||||
|
return withInput(t, r, meshOf())
|
||||||
|
}
|
||||||
|
|
||||||
|
// seats is the seat list a request carries, as the checker hands the one its reader holds (ADR 0635 §1, ADR 0636 §4).
|
||||||
|
func seats(t *testing.T) []json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
var out []json.RawMessage
|
||||||
|
for _, d := range seatlisttest.Definitions() {
|
||||||
|
raw, err := json.Marshal(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out = append(out, raw)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withInput is r with in as its input, as a caller puts it on the wire: the request's shape carries the input as JSON.
|
||||||
|
func withInput(t *testing.T, r composer.Request, in compose.Input) composer.Request {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &r.Input); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func runJSON(t *testing.T, args []string, in []byte) (int, composer.Verdict, string) {
|
||||||
|
t.Helper()
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
code := run(args, bytes.NewReader(in), &out, &errOut)
|
||||||
|
var v composer.Verdict
|
||||||
|
if code == 0 {
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||||
|
t.Fatalf("the verdict is not JSON: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return code, v, errOut.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func encode(t *testing.T, r composer.Request) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC1: the command reads the plain input and the manifests on standard input, validates every manifest, composes
|
||||||
|
// the machines named and writes one verdict: per machine composes or the reason, per manifest accepted or the field
|
||||||
|
// refused.
|
||||||
|
func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) {
|
||||||
|
r := request(t, composer.DepthResolve)
|
||||||
|
r.Cases[0].Manifests["later"] = composer.Manifest{Raw: json.RawMessage(`{"module":"later","version":"1","shares":["x"]}`)}
|
||||||
|
r.Cases[0].Manifests["refused"] = composer.Manifest{Raw: json.RawMessage(`{"module":"refused","version":"1"}`)}
|
||||||
|
r.Cases[0].Verdicts = bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"refused": {V: 1, Module: "refused", Judge: "src2:j", State: meshcatalogclient.VerdictRefused, Problems: []string{"it is wrong"}}})
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
if len(v.Cases) != 1 || v.Cases[0].Name != "trunk" || v.SDK == "" {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
c := v.Cases[0]
|
||||||
|
// ADR 0578 §5, §6: the composer judges nothing; it sets an unknown field aside and names it, and holds a manifest
|
||||||
|
// its verdict refuses, named in the indexer's words.
|
||||||
|
if got := c.Manifests["later"]; !got.Accepted || !strings.Contains(got.SetAside, `"shares"`) {
|
||||||
|
t.Errorf("the manifest with an unknown field reads %+v, want accepted naming the field set aside", got)
|
||||||
|
}
|
||||||
|
if got := c.Manifests["refused"]; got.Accepted || !strings.Contains(got.Refused, "refused the manifest of refused: it is wrong") {
|
||||||
|
t.Errorf("the refused manifest reads %+v, want refused in the verdict's words", got)
|
||||||
|
}
|
||||||
|
if got := c.Manifests["nats"]; !got.Accepted {
|
||||||
|
t.Errorf("nats reads %+v, want accepted", got)
|
||||||
|
}
|
||||||
|
if len(c.Machines) != 3 {
|
||||||
|
t.Fatalf("machines = %v, want the three named", c.Machines)
|
||||||
|
}
|
||||||
|
for name, m := range c.Machines {
|
||||||
|
if !m.Composes && m.Reason == "" {
|
||||||
|
t.Errorf("%s does not compose and says no reason", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1966, the group try of 2026-11-06 10:07: the composer judges against the seat list its request carries, the
|
||||||
|
// node-engine's from the bus (ADR 0635 §1). The uplink's holder steps back from the resolver file for the resolver's
|
||||||
|
// holder by the seat's definition; without the list nothing is judged (ADR 0636 §1), never "both declare the path".
|
||||||
|
resolverMachine := func(withSeats bool) composer.Machine {
|
||||||
|
r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, HoldingsFromClaims: true,
|
||||||
|
Machines: []string{"laptop"}, Cases: []composer.Case{{Name: "trunk", Manifests: captured(t)}}}
|
||||||
|
if withSeats {
|
||||||
|
r.Seats = seats(t)
|
||||||
|
}
|
||||||
|
r = withInput(t, r, compose.Input{Nodes: []compose.Node{{Name: "laptop",
|
||||||
|
Assigned: []string{"networkmanager", "systemd-resolved", "pacman", "systemd", "dnsmasq"},
|
||||||
|
Capabilities: map[string]bool{"file-references": true, "privileged": true, "service-manager": true, "package-manager": true, "uplink-networkmanager": true}}}})
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 || len(v.Cases) != 1 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
return v.Cases[0].Machines["laptop"]
|
||||||
|
}
|
||||||
|
if m := resolverMachine(true); strings.Contains(m.Reason+strings.Join(m.Problems, " "), "both declare") || m.NotJudged != "" {
|
||||||
|
t.Errorf("with the seat list, the uplink's and the resolver's holders on one machine: %+v, want the uplink's stepping back", m)
|
||||||
|
}
|
||||||
|
if m := resolverMachine(false); m.Composes || m.NotJudged != NoSeatList || m.Reason != "" || m.Blamed == nil {
|
||||||
|
t.Errorf("without a seat list: %+v, want not judged, saying why, blaming none", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Design 47 G §721, §733 (the node-engine's composer): a machine's verdict names the modules it blames, each with its
|
||||||
|
// problem, and always writes blamed (A-469h). On a node composed as its node-engine composes it (ComposedHow), blamed is
|
||||||
|
// the modules the change refuses that the node held neither in its read nor composing without the change (A-469j).
|
||||||
|
file := func(module, path, content string) string {
|
||||||
|
return `{"module": "` + module + `", "version": "1", "resources": [{"id": "f", "type": "file", "path": "` + path +
|
||||||
|
`", "mode": "0600", "content": "` + content + `"}]}`
|
||||||
|
}
|
||||||
|
plain := func(module string) string { return `{"module": "` + module + `", "version": "1"}` }
|
||||||
|
secret := `N=${secret:repository}\\n`
|
||||||
|
search := map[string]string{"m1": plain("m1"), "m2": file("m2", "/var/lib/m2/env", secret), "m3": plain("m3")}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
manifests map[string]string
|
||||||
|
assigned []string
|
||||||
|
how *compose.ComposedHow
|
||||||
|
tries int // the request's blame_tries
|
||||||
|
blamed []blame // each module blamed, its problem holding the words given
|
||||||
|
}{
|
||||||
|
// G-24 H-AC1: a fixture module whose manifest is refused on the machine is blamed by name.
|
||||||
|
{"a module the change refuses is blamed by name", map[string]string{"app": file("app", "/var/lib/app/env", `N=${secret:repository}\\n`)},
|
||||||
|
[]string{"app"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}}, 20, []blame{{Module: "app", Problem: "repository"}}},
|
||||||
|
// Held composing without the change (HeldBefore): left out of blamed.
|
||||||
|
{"a module held before the change is not blamed", map[string]string{"app": file("app", "/var/lib/app/env", `N=${secret:repository}\\n`)},
|
||||||
|
[]string{"app"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}, HeldBefore: map[string]string{"app": "held"}}, 0, []blame{}},
|
||||||
|
// G-24 H-AC2 (A-468l): two modules owning one path are both blamed, with that path.
|
||||||
|
{"two modules owning one path are both blamed", map[string]string{"one": file("one", "/etc/shared.conf", "x"), "two": file("two", "/etc/shared.conf", "y")},
|
||||||
|
[]string{"one", "two"}, &compose.ComposedHow{Node: "a", Builds: map[string]string{}}, 0,
|
||||||
|
[]blame{{Module: "one", Problem: `one and two both declare the path "/etc/shared.conf"`}, {Module: "two", Problem: `one and two both declare the path "/etc/shared.conf"`}}},
|
||||||
|
// The same, on a node whose input says not how it composes: nothing is held, both owners are blamed from the refusal.
|
||||||
|
{"two path owners, composed without how the node composes", map[string]string{"one": file("one", "/etc/shared.conf", "x"), "two": file("two", "/etc/shared.conf", "y")},
|
||||||
|
[]string{"one", "two"}, nil, 20,
|
||||||
|
[]blame{{Module: "one", Problem: `one and two both declare the path "/etc/shared.conf"`}, {Module: "two", Problem: `one and two both declare the path "/etc/shared.conf"`}}},
|
||||||
|
// G-24 H-AC5: a machine that fails with no module held is searched over the changed modules placed on it (here every
|
||||||
|
// one, the input naming no manifest its builds were made of, A-469k), in assignment order; the search ends at the
|
||||||
|
// first whose hold ends the problem, so m3 is never tried.
|
||||||
|
{"the search ends at the first module whose hold ends the problem", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 20,
|
||||||
|
[]blame{{Module: "m2", Problem: "repository"}}},
|
||||||
|
// A module whose case manifest is the one its build was made of is not changed (A-469k), so never held: m2's
|
||||||
|
// problem is then no changed module's, and none is blamed.
|
||||||
|
{"a module its build was made of is not tried", search, []string{"m1", "m2", "m3"},
|
||||||
|
&compose.ComposedHow{Node: "a", Manifests: map[string]string{"m2": digest(t, search["m2"])}}, 20, []blame{}},
|
||||||
|
// A-470v: a hold that changes the problem but leaves the machine not composing keeps that module held and blamed,
|
||||||
|
// and the search goes on against the retry's problem: both modules the change refuses are blamed.
|
||||||
|
{"every module the change refuses is blamed", map[string]string{"m1": file("m1", "/var/lib/m1/env", secret), "m2": file("m2", "/var/lib/m2/env", secret), "m3": plain("m3")},
|
||||||
|
[]string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 20, []blame{{Module: "m1", Problem: "repository"}, {Module: "m2", Problem: "repository"}}},
|
||||||
|
// G-24 H-AC4: past the request's blame_tries the modules not yet tried are named so.
|
||||||
|
{"past blame_tries the untried are named", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 1,
|
||||||
|
[]blame{{Module: "m2", Problem: composer.NotTried}, {Module: "m3", Problem: composer.NotTried}}},
|
||||||
|
// A-468i: a request with no blame_tries gets no search; the machine fails on the node's own account (H-AC3).
|
||||||
|
{"a request with no blame_tries is not searched", search, []string{"m1", "m2", "m3"}, &compose.ComposedHow{Node: "a"}, 0, []blame{}},
|
||||||
|
// A-469j: one search, the composer's, inside the node-engine's composition (ComposeSent): an input that says not how
|
||||||
|
// its node composes is composed without one, so its failure is the node's own.
|
||||||
|
{"a node that says not how it composes is not searched", search, []string{"m1", "m2", "m3"}, nil, 20, []blame{}},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, Machines: []string{"a"}, Seats: seats(t), BlameTries: row.tries,
|
||||||
|
Cases: []composer.Case{{Name: "change", Manifests: map[string]composer.Manifest{}}}}
|
||||||
|
for name, raw := range row.manifests {
|
||||||
|
r.Cases[0].Manifests[name] = composer.Manifest{Raw: json.RawMessage(raw)}
|
||||||
|
}
|
||||||
|
r = withInput(t, r, compose.Input{Nodes: []compose.Node{{Name: "a", Assigned: row.assigned}}, ComposedHow: row.how})
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if code := run(nil, bytes.NewReader(encode(t, r)), &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), `"blamed":`) {
|
||||||
|
t.Errorf("the verdict writes no blamed: %s", out.String())
|
||||||
|
}
|
||||||
|
var v composer.Verdict
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := v.Cases[0].Machines["a"]
|
||||||
|
if held := row.how != nil && row.how.HeldBefore != nil; m.Composes != held {
|
||||||
|
t.Errorf("composes = %v, want %v: %+v", m.Composes, held, m)
|
||||||
|
}
|
||||||
|
if len(m.Blamed) != len(row.blamed) {
|
||||||
|
t.Fatalf("blamed %+v, want %+v (%+v)", m.Blamed, row.blamed, m)
|
||||||
|
}
|
||||||
|
for i, b := range row.blamed {
|
||||||
|
if m.Blamed[i].Module != b.Module || !strings.Contains(m.Blamed[i].Problem, b.Problem) {
|
||||||
|
t.Errorf("blamed %+v, want %+v", m.Blamed, row.blamed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// digest is the content digest of a manifest (compose.ContentDigest).
|
||||||
|
func digest(t *testing.T, raw string) string {
|
||||||
|
t.Helper()
|
||||||
|
d, err := compose.ContentDigest([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// blame is one module a machine's verdict blames (composer.Machine.Blamed).
|
||||||
|
type blame = struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Problem string `json:"problem"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC3: over the captured catalogue and a mesh of three machines, the command's verdict equals, machine for
|
||||||
|
// machine and manifest for manifest, what the linked reader, resolution and composition answer, at both depths.
|
||||||
|
func TestTheCommandAnswersWhatTheLinkedCompositionAnswers(t *testing.T) {
|
||||||
|
for _, depth := range []string{composer.DepthResolve, composer.DepthCompose} {
|
||||||
|
r := request(t, depth)
|
||||||
|
code, v, stderr := runJSON(t, nil, encode(t, r))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("%s: exit %d: %s", depth, code, stderr)
|
||||||
|
}
|
||||||
|
in := meshOf()
|
||||||
|
in.Catalogue = map[string]catalogue.Manifest{}
|
||||||
|
for name, m := range r.Cases[0].Manifests {
|
||||||
|
// A request without a list of predefined keys reads a fallback as the indexer does (Request.Predefined).
|
||||||
|
parsed, err := catalogue.Decode(m.Raw)
|
||||||
|
got := v.Cases[0].Manifests[name]
|
||||||
|
if (err == nil) != got.Accepted || (err != nil && err.Error() != got.Refused) {
|
||||||
|
t.Errorf("%s: the manifest of %s: linked %v, command %+v", depth, name, err, got)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
in.Catalogue[name] = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in.Holdings = composer.HoldingsOf(in.Catalogue, in.Nodes)
|
||||||
|
composed := 0
|
||||||
|
for _, node := range r.Machines {
|
||||||
|
got := v.Cases[0].Machines[node]
|
||||||
|
var want composer.Machine
|
||||||
|
if depth == composer.DepthResolve {
|
||||||
|
if _, err := compose.Resolve(in, node); err != nil {
|
||||||
|
want = composer.Machine{Reason: err.Error()}
|
||||||
|
} else {
|
||||||
|
want = composer.Machine{Composes: true}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
d, problems, err := compose.Compose(in, node)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
want = composer.Machine{Reason: err.Error()}
|
||||||
|
default:
|
||||||
|
want = composer.Machine{Composes: len(problems) == 0, Problems: problems, Refusals: d.Refusals()}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
want.Reason = "the node-engine's validator refuses the declaration"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// G-24 H-AC3 (§721): a machine that does not compose and blames no module (ace: its own sealing key) fails on the
|
||||||
|
// node's own account, blamed written empty.
|
||||||
|
want.Blamed = []blame{}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("%s: %s: command %+v, linked %+v", depth, node, got, want)
|
||||||
|
}
|
||||||
|
if got.Composes {
|
||||||
|
composed++
|
||||||
|
} else {
|
||||||
|
t.Logf("%s: %s does not compose: %s %v", depth, node, got.Reason, got.Problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("%s: %d of %d machines compose", depth, composed, len(r.Machines))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621 note 6: a person composes a catalogue checkout (repository@ref checked out or archived: the command fetches
|
||||||
|
// nothing) for a node and reads its declaration: each path with its kind, owner and mode.
|
||||||
|
func TestACatalogueCheckoutIsComposedForANode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "modules", "demo"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifest := `{"module": "demo", "version": "1", "resources": [{"id": "state", "type": "directory", ` +
|
||||||
|
`"path": "/var/lib/demo", "mode": "0750", "owner": "root"}]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "modules", "demo", "module.json"), []byte(manifest), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
in := encode(t, withInput(t, composer.Request{V: composer.Version, Seats: seats(t)}, compose.Input{Nodes: []compose.Node{{Name: "laptop", Assigned: []string{"demo"}}}}))
|
||||||
|
code, v, stderr := runJSON(t, []string{"-catalogue", dir, "-node", "laptop", "-declarations"}, in)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
m := v.Cases[0].Machines["laptop"]
|
||||||
|
want := []composer.Path{{Path: "/var/lib/demo", Kind: "directory", Owner: "root", Mode: "0750", Module: ""}}
|
||||||
|
if !m.Composes || len(m.Declared) != 1 || m.Declared[0].Path != want[0].Path || m.Declared[0].Mode != "0750" ||
|
||||||
|
m.Declared[0].Owner != "root" || m.Declared[0].Kind != "directory" {
|
||||||
|
t.Fatalf("laptop = %+v, want it composed declaring %+v", m, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A run step hands the command no standard input: the request is read from the file -request names, or from the one
|
||||||
|
// MESH_RUN_INPUT names, and a file that is not there is a request it cannot read.
|
||||||
|
func TestTheRequestIsReadFromAFile(t *testing.T) {
|
||||||
|
file := filepath.Join(t.TempDir(), "request.json")
|
||||||
|
r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{
|
||||||
|
"demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}}
|
||||||
|
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, v, stderr := runJSON(t, []string{"-request", file}, nil)
|
||||||
|
if code != 0 || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
|
||||||
|
t.Fatalf("exit %d (%s), verdict %+v", code, stderr, v)
|
||||||
|
}
|
||||||
|
if code, _, stderr := runJSON(t, []string{"-request", file + ".gone"}, nil); code != 1 || !strings.Contains(stderr, "cannot be read") {
|
||||||
|
t.Errorf("a request file that is not there: exit %d, %q", code, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pipeline step names its input file in MESH_RUN_INPUT: the command reads the request there and the verdict is the
|
||||||
|
// last line of standard output (design 47 G A-167).
|
||||||
|
r.Depth = composer.DepthCompose
|
||||||
|
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_RUN_INPUT", file)
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
||||||
|
v = composer.Verdict{}
|
||||||
|
if err := json.Unmarshal([]byte(lines[len(lines)-1]), &v); err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
|
||||||
|
t.Fatalf("last line is no verdict: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_RUN_INPUT", file+".gone")
|
||||||
|
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "cannot be read") {
|
||||||
|
t.Errorf("a missing input file: exit %d, %q", code, errOut.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
|
||||||
|
func TestTheExitCodesSayWhatFailed(t *testing.T) {
|
||||||
|
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
in string
|
||||||
|
code int
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"not JSON", nil, "{", 1, "not JSON"},
|
||||||
|
{"another version", nil, `{"v":2}`, 1, "version 2"},
|
||||||
|
{"a digest left unfilled", nil, string(unfilled), 1, "digest"},
|
||||||
|
{"an unknown depth", nil, `{"v":1,"depth":"deep"}`, 1, "depth"},
|
||||||
|
{"nothing to read", nil, "", 1, "no request"},
|
||||||
|
{"a wrong flag", []string{"-nope"}, "", 2, "nope"},
|
||||||
|
{"an argument", []string{"extra"}, "", 2, "unexpected"},
|
||||||
|
} {
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if got := run(c.args, strings.NewReader(c.in), &out, &errOut); got != c.code || !strings.Contains(errOut.String(), c.says) {
|
||||||
|
t.Errorf("%s: exit %d saying %q, want %d saying %q", c.name, got, errOut.String(), c.code, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownPackages is every package of this repository the command imports, directly or not, with the packages each
|
||||||
|
// imports: what this repository adds to the composition; mesh-sdk's packages are judged in mesh-sdk.
|
||||||
|
func ownPackages(t *testing.T) map[string][]string {
|
||||||
|
t.Helper()
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
root := filepath.Join(filepath.Dir(file), "..", "..")
|
||||||
|
const module = "github.com/novox/mesh-host"
|
||||||
|
out := map[string][]string{}
|
||||||
|
var walk func(path string)
|
||||||
|
walk = func(path string) {
|
||||||
|
if _, done := out[path]; done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rel := strings.TrimPrefix(strings.TrimPrefix(path, module), "/")
|
||||||
|
pkg, err := build.Default.ImportDir(filepath.Join(root, rel), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", path, err)
|
||||||
|
}
|
||||||
|
out[path] = pkg.Imports
|
||||||
|
for _, imp := range pkg.Imports {
|
||||||
|
if strings.HasPrefix(imp, module+"/") {
|
||||||
|
walk(imp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(composer.Package)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard
|
||||||
|
// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types) and catalog key codec (a catalog entry's key, design 47 G A-144), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and
|
||||||
|
// generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or
|
||||||
|
// another process.
|
||||||
|
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
|
||||||
|
forbidden := []string{"net/http", "net/rpc", "net/smtp", "os/exec", "database/sql", "plugin", "syscall"}
|
||||||
|
own := ownPackages(t)
|
||||||
|
if len(own) < 4 {
|
||||||
|
t.Fatalf("the command reaches %d packages of this repository; the walk found too few to judge", len(own))
|
||||||
|
}
|
||||||
|
for path, imports := range own {
|
||||||
|
for _, imp := range imports {
|
||||||
|
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
|
||||||
|
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") {
|
||||||
|
t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp)
|
||||||
|
}
|
||||||
|
for _, f := range forbidden {
|
||||||
|
if imp == f {
|
||||||
|
t.Errorf("%s imports %s", path, imp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1621/AC2: run with an empty network namespace, the command still writes its verdict: it needs no network. The test
|
||||||
|
// runs its own binary again under `unshare --net` as an unprivileged user's namespace; where the kernel or the sandbox
|
||||||
|
// allows no new namespace (a build module's run step is already one with no network), it says so and skips.
|
||||||
|
func TestTheCommandRunsWithNoNetwork(t *testing.T) {
|
||||||
|
if os.Getenv("MESH_COMPOSE_HELPER") == "1" {
|
||||||
|
os.Exit(run(nil, os.Stdin, os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
|
unshare, err := exec.LookPath("unshare")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("no unshare on this machine, so no empty network namespace can be made here")
|
||||||
|
}
|
||||||
|
if out, err := exec.Command(unshare, "--net", "--map-root-user", "true").CombinedOutput(); err != nil {
|
||||||
|
t.Skipf("this machine makes no unprivileged network namespace (%v: %s); a run step's own has no network", err, out)
|
||||||
|
}
|
||||||
|
r := request(t, composer.DepthResolve)
|
||||||
|
cmd := exec.Command(unshare, "--net", "--map-root-user", os.Args[0], "-test.run=^TestTheCommandRunsWithNoNetwork$")
|
||||||
|
cmd.Env = []string{"MESH_COMPOSE_HELPER=1"}
|
||||||
|
cmd.Stdin = bytes.NewReader(encode(t, r))
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
cmd.Stdout, cmd.Stderr = &out, &errOut
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
t.Fatalf("the command in an empty network namespace: %v: %s", err, errOut.String())
|
||||||
|
}
|
||||||
|
var v composer.Verdict
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 1 || len(v.Cases[0].Machines) != 3 {
|
||||||
|
t.Fatalf("no verdict from the command with no network: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(v.Cases[0].Machines))
|
||||||
|
for n := range v.Cases[0].Machines {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
t.Logf("composed with no network: %v", names)
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/compositions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the
|
||||||
|
// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs:
|
||||||
|
// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran
|
||||||
|
// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a
|
||||||
|
// 0700 directory: a composition carries values sealed to this machine.
|
||||||
|
type appliedStore struct{ dir string }
|
||||||
|
|
||||||
|
func appliedStoreFor(statePath string) appliedStore {
|
||||||
|
return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path.
|
||||||
|
var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
|
||||||
|
|
||||||
|
func (s appliedStore) path(module string) (string, error) {
|
||||||
|
if !appliedName.MatchString(module) || strings.Contains(module, "..") {
|
||||||
|
return "", fmt.Errorf("%q is not a module name the applied store takes", module)
|
||||||
|
}
|
||||||
|
return filepath.Join(s.dir, module+".json"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds says whether the store keeps the module's composition.
|
||||||
|
func (s appliedStore) holds(module string) bool {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err = os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Load() (map[string][]byte, error) {
|
||||||
|
entries, err := os.ReadDir(s.dir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
var errs []error
|
||||||
|
for _, e := range entries {
|
||||||
|
module, ok := strings.CutSuffix(e.Name(), ".json")
|
||||||
|
if !ok || e.IsDir() || !appliedName.MatchString(module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(s.dir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[module] = raw
|
||||||
|
}
|
||||||
|
return out, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Keep(module string, raw []byte) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(s.dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(s.dir, "."+module+".*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmp.Name())
|
||||||
|
if _, err := tmp.Write(raw); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp.Name(), path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Drop(module string) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownModuleApplied is the module of the composition the store holds at the fingerprint fp, the engine's own build's
|
||||||
|
// (design 47 G §527, A-149): "" when fp is none or no record holds it.
|
||||||
|
func ownModuleApplied(s appliedStore, fp string) string {
|
||||||
|
if fp == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
all, _ := s.Load()
|
||||||
|
for module, raw := range all {
|
||||||
|
var c compositions.Composed
|
||||||
|
if module != agent.NodeModule && json.Unmarshal(raw, &c) == nil && c.FP == fp && c.Module != "" {
|
||||||
|
return c.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An apply runs in a context the bus's loss cannot end: the link's context ends when the bus goes, and the bus may go
|
||||||
|
// because of the apply itself (it replaces the bus's own container), which then must finish (issue 1945, live on
|
||||||
|
// 2026-10-25 at 11:23: novox's node-engine removed the bus container, its link's context ended, and the docker run that
|
||||||
|
// would have made the new one was never made).
|
||||||
|
func TestAnApplyOutlivesTheLinksContext(t *testing.T) {
|
||||||
|
type key struct{}
|
||||||
|
link, cancel := context.WithCancel(context.WithValue(context.Background(), key{}, "v"))
|
||||||
|
ctx, done := applyContext(link)
|
||||||
|
defer done()
|
||||||
|
cancel()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
t.Fatalf("the apply's context ended with the link's: %v", ctx.Err())
|
||||||
|
}
|
||||||
|
if ctx.Value(key{}) != "v" {
|
||||||
|
t.Fatal("the apply's context lost the link's values")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An apply is bounded all the same: a step that hangs (a docker call that never returns) cannot hold the node agent for
|
||||||
|
// ever (the review of issue 1945).
|
||||||
|
func TestAnApplyIsBounded(t *testing.T) {
|
||||||
|
t.Run("the marks of the haves never hold the pass", marksOfTheHavesNeverHoldThePass)
|
||||||
|
ctx, done := applyContext(context.Background())
|
||||||
|
defer done()
|
||||||
|
at, ok := ctx.Deadline()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("an apply has no bound")
|
||||||
|
}
|
||||||
|
if left := time.Until(at); left > applyBound || left < applyBound-time.Minute {
|
||||||
|
t.Fatalf("the bound is %s away, want about %s", left, applyBound)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// applyAndKeep applies a whole declaration the way the carried bundle is applied, and says what came of it as the facts
|
||||||
|
// the node says: what the tests of the apply's steps read. The node-engine applies no whole declaration on its own now
|
||||||
|
// (novox/hq ADR 0539 §9); this is the test's way to put a machine in a state.
|
||||||
|
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler,
|
||||||
|
say link.Announce) link.Report {
|
||||||
|
declared, err := declaration.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return link.Report{Refused: err.Error()}
|
||||||
|
}
|
||||||
|
got := applyModuleHeld(ctx, opts, declared, nil, sched, say)
|
||||||
|
if got.Refused != "" {
|
||||||
|
return link.Report{Refused: got.Refused}
|
||||||
|
}
|
||||||
|
held := map[string]bool{}
|
||||||
|
for _, h := range got.Updated.Held {
|
||||||
|
held[h.ID] = true
|
||||||
|
}
|
||||||
|
report := factsOf(ctx, opts, declared, got.Updated, got.Outcome, held)
|
||||||
|
if signed != nil {
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil {
|
||||||
|
report.Failed = map[string]string{"keeping the declaration": err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got.Err != nil {
|
||||||
|
report.Failed = map[string]string{"apply": got.Err.Error()}
|
||||||
|
}
|
||||||
|
report.Failed = withUnanswered(report.Failed, got.Probe)
|
||||||
|
report.Unapplied = unapplied(declared, got.Outcome, got.Err)
|
||||||
|
return report
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// authorityPath is where this engine keeps the mesh's own certificate as it last applied it (public): beside the
|
||||||
|
// certificate issued for the machine's name, read into its record (novox/hq ADR 0555 §3).
|
||||||
|
func authorityPath(state string) string { return filepath.Join(filepath.Dir(state), "authority.crt") }
|
||||||
|
|
||||||
|
// keepAuthority keeps the mesh's own certificate a composition carries (the resource a module's certificate.authority
|
||||||
|
// asks for, catalogue.AuthorityID), so the record says the authority this machine holds. A composition carrying none
|
||||||
|
// changes nothing.
|
||||||
|
func keepAuthority(state string, c engcomp.Composed) error {
|
||||||
|
for _, raw := range c.Resources {
|
||||||
|
var r struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(raw, &r) != nil || r.Content == "" || !strings.HasSuffix(r.ID, "."+catalogue.AuthorityID()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := authorityPath(state)
|
||||||
|
if was, err := os.ReadFile(path); err == nil && string(was) == r.Content {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(path, []byte(r.Content), 0o644)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/pem"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||||
|
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
|
||||||
|
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
|
||||||
|
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||||
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
|
res := func(id, content string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": id, "type": "file", "path": "/etc/x", "content": content})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
if err := keepAuthority(state, engcomp.Composed{Resources: []json.RawMessage{res("web.conf", "x")}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(authorityPath(state)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("a composition with no authority kept one: %v", err)
|
||||||
|
}
|
||||||
|
if err := keepAuthority(state, engcomp.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
|
||||||
|
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1126, A6 (ADR 0550 §2; the controller ends by its unassignment): a node enrolled with the controller's key
|
||||||
|
// trusts it never, whatever the lease says; before the vault's signers are read it trusts nothing, once read the
|
||||||
|
// vault's key alone, and the record names the vault's alone. Each change tells the record's writer.
|
||||||
|
controllerPub, controller, _ := ed25519.GenerateKey(nil)
|
||||||
|
vaultPub, vault, _ := ed25519.GenerateKey(nil)
|
||||||
|
trust := newMeshTrust()
|
||||||
|
told := 0
|
||||||
|
trust.onChange(func() { told++ })
|
||||||
|
keys := recordKeys(state, trust.Trusts)
|
||||||
|
ctrlFP, vaultFP := meshvaultclient.SignerFingerprint(controllerPub), meshvaultclient.SignerFingerprint(vaultPub)
|
||||||
|
enrol, _ := meshvaultclient.SignContext(meshvaultclient.KindEnrolment)
|
||||||
|
msg := []byte("a reply")
|
||||||
|
if got := keys().Trusts; len(got) != 0 {
|
||||||
|
t.Errorf("before the signers are read the record trusts %v", got)
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(controller, enrol, msg)); err == nil {
|
||||||
|
t.Error("a controller-signed message is accepted")
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(vault, enrol, msg)); err == nil {
|
||||||
|
t.Error("a vault-signed message is accepted before the vault's key is read")
|
||||||
|
}
|
||||||
|
vaultCA := testAuthority(t, vaultPub, vault)
|
||||||
|
keyValue, _ := json.Marshal(meshvaultclient.NewSignerKey(vaultPub, time.Now()))
|
||||||
|
fp, _ := meshvaultclient.AuthorityFingerprint(vaultCA)
|
||||||
|
authValue, _ := json.Marshal(meshvault.SignersByAuthorityValue{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
|
||||||
|
trust.heardSigner(meshvault.SignersByKeyKey(), keyValue, false, func(string) {})
|
||||||
|
trust.heardSigner(meshvault.SignersByAuthorityKey(), authValue, false, func(string) {})
|
||||||
|
if s, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(vault, enrol, msg)); err != nil || s.Name != meshvaultclient.SignerVault {
|
||||||
|
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
|
||||||
|
}
|
||||||
|
if _, err := trust.Verify(meshvaultclient.KindEnrolment, msg, meshvaultclient.SignMessage(controller, enrol, msg)); err == nil {
|
||||||
|
t.Error("a controller-signed message is accepted once the vault's key is read")
|
||||||
|
}
|
||||||
|
if got := keys().Trusts; strings.Contains(strings.Join(got, " "), ctrlFP) || !meshvaultclient.TrustsKey(got, meshvaultclient.SignerVault, vaultFP) ||
|
||||||
|
trust.VaultAuthority() != vaultCA || told != 2 {
|
||||||
|
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
||||||
|
}
|
||||||
|
// A signers' key naming another fingerprint is not trusted.
|
||||||
|
bad, _ := json.Marshal(meshvault.SignersByKeyValue{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
|
||||||
|
trust.heardSigner(meshvault.SignersByKeyKey(), bad, false, func(string) {})
|
||||||
|
if !meshvaultclient.TrustsKey(keys().Trusts, meshvaultclient.SignerVault, vaultFP) {
|
||||||
|
t.Error("a signers' key that does not check replaced the vault's")
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
|
||||||
|
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
|
||||||
|
serving, err := identity.GenerateServingKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
signedBy := func(ca string, key ed25519.PrivateKey) string {
|
||||||
|
parent, _ := meshvaultclient.ParseCertificate(ca)
|
||||||
|
der, err := x509.CreateCertificate(nil, meshvaultclient.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
|
||||||
|
parent, ed25519.PublicKey(servingPub), key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
|
}
|
||||||
|
var asked []any
|
||||||
|
answer := signedBy(vaultCA, vault)
|
||||||
|
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
|
||||||
|
asked = append(asked, verb, args)
|
||||||
|
return json.Marshal(meshvault.SignAnswer{Kind: meshvaultclient.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||||
|
}
|
||||||
|
written := 0
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != meshvault.VerbSign {
|
||||||
|
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
|
||||||
|
}
|
||||||
|
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
|
||||||
|
t.Errorf("a certificate of another authority was kept: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory,
|
||||||
|
// written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path.
|
||||||
|
t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) {
|
||||||
|
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
|
||||||
|
if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") {
|
||||||
|
t.Fatalf("an empty store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
module string
|
||||||
|
takes bool
|
||||||
|
}{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} {
|
||||||
|
if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes {
|
||||||
|
t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(applied.dir)
|
||||||
|
dirInfo, _ := os.Stat(applied.dir)
|
||||||
|
fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json"))
|
||||||
|
if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode())
|
||||||
|
}
|
||||||
|
if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") {
|
||||||
|
t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err)
|
||||||
|
}
|
||||||
|
if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 {
|
||||||
|
t.Fatalf("the store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
|
||||||
|
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
|
||||||
|
t.Helper()
|
||||||
|
tpl := meshvaultclient.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
|
||||||
|
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "ok": ok, "at": at, "error": why})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup is done when the holder says so for this node and module, held on its words when it says it failed, and held
|
||||||
|
// when it says nothing within the bound; another node's or module's event, or one from before the ask, is not the answer
|
||||||
|
// (R6-A A5, novox/hq ADR 0429 §14).
|
||||||
|
func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
||||||
|
since := time.Now()
|
||||||
|
run := func(within time.Duration, events ...[]byte) (done bool, words string) {
|
||||||
|
ch := make(chan []byte, len(events))
|
||||||
|
for _, e := range events {
|
||||||
|
ch <- e
|
||||||
|
}
|
||||||
|
got := awaitBackup(context.Background(), ch, "n1", "db", since, within)
|
||||||
|
return got.Done, got.Words
|
||||||
|
}
|
||||||
|
if done, _ := run(time.Second,
|
||||||
|
backupEvent("n2", "db", true, since, ""), backupEvent("n1", "web", true, since, ""),
|
||||||
|
backupEvent("n1", "db", false, since.Add(-time.Hour), "an old night"),
|
||||||
|
backupEvent("n1", "db", true, since.Add(time.Second), "")); !done {
|
||||||
|
t.Error("the event of this node and module was not the answer")
|
||||||
|
}
|
||||||
|
if done, words := run(time.Second, backupEvent("n1", "db", false, since, "the repository is locked")); done ||
|
||||||
|
!strings.Contains(words, "repository is locked") {
|
||||||
|
t.Errorf("a failed backup: done=%v words=%q", done, words)
|
||||||
|
}
|
||||||
|
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
||||||
|
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
||||||
|
}
|
||||||
|
// The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs
|
||||||
|
// the module up again (postgres on novox, 2026-10-31).
|
||||||
|
stopped, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted ||
|
||||||
|
got.Words != agent.BackupInterrupted {
|
||||||
|
t.Errorf("a stopped node agent: %+v", got)
|
||||||
|
}
|
||||||
|
if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted {
|
||||||
|
t.Errorf("silence within the bound taken as interrupted: %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
|
||||||
|
// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
|
||||||
|
// reached it is never asked twice.
|
||||||
|
notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
|
||||||
|
"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
answers []error
|
||||||
|
within time.Duration
|
||||||
|
asked int
|
||||||
|
words string
|
||||||
|
}{
|
||||||
|
{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
|
||||||
|
// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
|
||||||
|
{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
|
||||||
|
0, "times over"},
|
||||||
|
{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
|
||||||
|
time.Second, 1, "deadline exceeded"},
|
||||||
|
{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
|
||||||
|
} {
|
||||||
|
asked := 0
|
||||||
|
_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
|
||||||
|
asked++
|
||||||
|
return json.RawMessage(`{}`), row.answers[asked-1]
|
||||||
|
}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
|
||||||
|
if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
|
||||||
|
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
|
||||||
|
// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
|
||||||
|
told := &settledRecorder{}
|
||||||
|
events := make(chan []byte, 4)
|
||||||
|
for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
|
||||||
|
events <- []byte(body)
|
||||||
|
}
|
||||||
|
linked, end := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for len(told.heard()) < 2 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
end()
|
||||||
|
<-done
|
||||||
|
if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
|
||||||
|
t.Errorf("the agent was told %q settled, want distribution,gitea", got)
|
||||||
|
}
|
||||||
|
// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
|
||||||
|
// one (ENG-8, ENG-10).
|
||||||
|
d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
|
||||||
|
`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pull := func(image string) error {
|
||||||
|
return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
|
||||||
|
t.Errorf("a store's image is said as %+v", got)
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
|
||||||
|
t.Errorf("a public image is said as %+v", got)
|
||||||
|
}
|
||||||
|
if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
|
||||||
|
t.Errorf("a failure that is no pull is said as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledRecorder is the agent a hearing tells.
|
||||||
|
type settledRecorder struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
list []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *settledRecorder) Settled(holder string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.list = append(r.list, holder)
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *settledRecorder) heard() []string {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return append([]string(nil), r.list...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a
|
||||||
|
// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to
|
||||||
|
// back up on this machine"). What node-backup.backed-up answers decides it.
|
||||||
|
func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
said string
|
||||||
|
nothing bool
|
||||||
|
}{
|
||||||
|
{`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true},
|
||||||
|
{`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true},
|
||||||
|
{`[]`, true},
|
||||||
|
{`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false},
|
||||||
|
} {
|
||||||
|
nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis")
|
||||||
|
if !ok || nothing != c.nothing {
|
||||||
|
t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok {
|
||||||
|
t.Error("an answer that is not the list was read")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// While a container waits, a pass is asked for every so often, not at some long interval; with nothing waiting, none
|
||||||
|
// (novox/hq ADR 0335 §2, ADR 0360 §11).
|
||||||
|
func TestAWaitIsAskedAgainSoon(t *testing.T) {
|
||||||
|
if BusyAskEvery > time.Minute {
|
||||||
|
t.Fatalf("a waiting container is asked every %s", BusyAskEvery)
|
||||||
|
}
|
||||||
|
var asked atomic.Int32
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
busyWaiting.Store(false)
|
||||||
|
t.Cleanup(func() { busyWaiting.Store(false) })
|
||||||
|
go busyPasses(ctx, func() { asked.Add(1) }, 10*time.Millisecond)
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if n := asked.Load(); n != 0 {
|
||||||
|
t.Fatalf("%d passes were asked with nothing waiting", n)
|
||||||
|
}
|
||||||
|
busyWaiting.Store(true)
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for asked.Load() == 0 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if asked.Load() == 0 {
|
||||||
|
t.Fatal("no pass was asked while a container waits")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A composition that names a file the provisioner fills without saying it is filled would have this node-engine write
|
||||||
|
// that file empty: its apply is refused, loudly, before anything is written (issue 1945, live on 2026-10-25 at 01:22,
|
||||||
|
// when ace, g14 and shanks wrote claude-code's, node-env's and power's files empty).
|
||||||
|
func TestAFileItCannotProduceIsNeverWrittenEmpty(t *testing.T) {
|
||||||
|
unit := json.RawMessage(`{"id":"power.after-boot-unit","type":"file","path":"/etc/systemd/system/mesh-power-after-boot.service","mode":"0644","owner-at-runtime":"owned"}`)
|
||||||
|
c := engcomp.Composed{Node: "ace", Resources: []json.RawMessage{unit}}
|
||||||
|
d, err := declarationOf(c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := refuseEmptyWrites(d); err == nil || !strings.Contains(err.Error(), "power.after-boot-unit") {
|
||||||
|
t.Fatalf("a file the provisioner fills, not filled here: %v", err)
|
||||||
|
}
|
||||||
|
c.Filled = []string{"/etc/systemd/system/mesh-power-after-boot.service"}
|
||||||
|
if d, _ = declarationOf(c); refuseEmptyWrites(d) != nil {
|
||||||
|
t.Fatalf("a filled file was refused: %v", refuseEmptyWrites(d))
|
||||||
|
}
|
||||||
|
plain := engcomp.Composed{Node: "ace", Resources: []json.RawMessage{json.RawMessage(
|
||||||
|
`{"id":"x.f","type":"file","path":"/etc/x","mode":"0644","content":"x"}`)}}
|
||||||
|
if d, _ = declarationOf(plain); refuseEmptyWrites(d) != nil {
|
||||||
|
t.Fatalf("a file with its content was refused: %v", refuseEmptyWrites(d))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,457 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||||
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take
|
||||||
|
// would set and what the disk holds now, and whether this machine is root-free.
|
||||||
|
|
||||||
|
// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and
|
||||||
|
// group, its mode, and which file it is.
|
||||||
|
func diskOf(info os.FileInfo) string {
|
||||||
|
owner := "unknown"
|
||||||
|
if uid, gid, ok := apply.OwnerOf(info); ok {
|
||||||
|
owner = fmt.Sprintf("%d:%d", uid, gid)
|
||||||
|
}
|
||||||
|
which := ""
|
||||||
|
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||||
|
which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the
|
||||||
|
// declared owner and mode it will set, and the disk as it is.
|
||||||
|
func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) {
|
||||||
|
return func(path string) (link.Judged, error) {
|
||||||
|
found, info, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, err
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info),
|
||||||
|
Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleOf(id string) string {
|
||||||
|
if at := strings.LastIndex(id, "."); at > 0 {
|
||||||
|
return id[:at]
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this
|
||||||
|
// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is.
|
||||||
|
func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) {
|
||||||
|
return func(act, access string) (link.Judged, error) {
|
||||||
|
d, err := declaredNow(statePath, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err)
|
||||||
|
}
|
||||||
|
for _, l := range d.Libraries {
|
||||||
|
if apply.LibraryAccess(l.Library) != access {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !l.Home {
|
||||||
|
return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+
|
||||||
|
"taken on its home", l.Library)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
|
||||||
|
defer cancel()
|
||||||
|
if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+
|
||||||
|
"(it is judged %s), so there is nothing to take", l.Library, v.Kind)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(l.Root)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err)
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: access, Path: filepath.Clean(l.Root),
|
||||||
|
Sets: "the group " + l.Group + ", group write and setgid on everything under its root",
|
||||||
|
Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil
|
||||||
|
}
|
||||||
|
return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563
|
||||||
|
// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again
|
||||||
|
// by hand.
|
||||||
|
func askTakes(acts *link.HandActs, waiting []string, say func(string)) {
|
||||||
|
if acts == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, access := range waiting {
|
||||||
|
if acts.AskedFor(link.ActAccess, access) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access,
|
||||||
|
"its root holds data outside its group", true); err != nil && say != nil {
|
||||||
|
say("the take of " + access + " could not be asked: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastAccounts is the newest look at every account (the health loop's), which root-free judges from.
|
||||||
|
var lastAccounts atomic.Pointer[accounts.Statement]
|
||||||
|
|
||||||
|
// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor).
|
||||||
|
const rootFreshFor = 15 * time.Minute
|
||||||
|
|
||||||
|
// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most
|
||||||
|
// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such
|
||||||
|
// account declared, and the login shell's execute not served here. Anything else is no pass, and says why.
|
||||||
|
func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict {
|
||||||
|
v := link.RootVerdict{Machine: node, Judged: now.UTC()}
|
||||||
|
var not []string
|
||||||
|
confined := ""
|
||||||
|
switch {
|
||||||
|
case st == nil:
|
||||||
|
not = append(not, "this machine's engine has judged no account yet")
|
||||||
|
case now.Sub(st.At) > rootFreshFor:
|
||||||
|
not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+
|
||||||
|
"minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339),
|
||||||
|
int(rootFreshFor.Minutes())))
|
||||||
|
default:
|
||||||
|
var named []string
|
||||||
|
for _, a := range st.Accounts {
|
||||||
|
if !a.Root {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
named = append(named, a.Name)
|
||||||
|
if a.State != accounts.Healthy {
|
||||||
|
not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(named) == 0 {
|
||||||
|
not = append(not, "no account here is declared never to become root, so agents run as the operator's "+
|
||||||
|
"account, which may become root")
|
||||||
|
} else if len(not) == 0 {
|
||||||
|
confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " +
|
||||||
|
st.At.UTC().Format(time.RFC3339) + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served {
|
||||||
|
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+
|
||||||
|
"can become root ("+servedWhy+")")
|
||||||
|
}
|
||||||
|
if len(not) > 0 {
|
||||||
|
v.Why = strings.Join(not, "; ")
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
v.Free, v.Why = true, confined+"; the login shell's execute is not served here"
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNoneKnownWord(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "is not judged"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's
|
||||||
|
// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery.
|
||||||
|
const settingFileSuffix = ".tools-conf"
|
||||||
|
|
||||||
|
// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served
|
||||||
|
// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied
|
||||||
|
// here says anything but withhold or cannot be read. why says which.
|
||||||
|
func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) {
|
||||||
|
var why []string
|
||||||
|
if heard == nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here")
|
||||||
|
} else if h, err := heard(); err != nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error())
|
||||||
|
} else if h {
|
||||||
|
why = append(why, "the bus hears execute answered here")
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
why = append(why, "what this machine applied could not be read: "+err.Error())
|
||||||
|
}
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if value, err := settingIn(r.Target); err != nil {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error())
|
||||||
|
} else if value != "withhold" {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(why) > 0, strings.Join(why, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingIn is the value of execute in a setting file, "" when it says none.
|
||||||
|
func settingIn(path string) (string, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
value := ""
|
||||||
|
lines := bufio.NewScanner(f)
|
||||||
|
for lines.Scan() {
|
||||||
|
t := strings.TrimSpace(lines.Text())
|
||||||
|
if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" {
|
||||||
|
value = strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
value = "not said"
|
||||||
|
}
|
||||||
|
return value, lines.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8).
|
||||||
|
func keepWaiting(vs []link.ResourceHealth) []string {
|
||||||
|
var out []string
|
||||||
|
for _, v := range vs {
|
||||||
|
for _, w := range v.AccessWaits {
|
||||||
|
if w.Reason == link.WaitKeep {
|
||||||
|
out = append(out, w.Access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's
|
||||||
|
// signed asks, its asks to the operator over the session's link, its acts kept beside the state.
|
||||||
|
func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs {
|
||||||
|
state := link.BusState{Session: queue, Node: node}
|
||||||
|
h := &link.HandActs{Node: node, State: state,
|
||||||
|
JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state),
|
||||||
|
JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||||
|
JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running,
|
||||||
|
RootFree: func() link.RootVerdict {
|
||||||
|
served, why := executeServedHere(opts.state, func() (bool, error) {
|
||||||
|
return queue.HeardServed("node-login-shell", "execute")
|
||||||
|
})
|
||||||
|
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
|
||||||
|
},
|
||||||
|
ManifestReads: queue.StatedReads, Say: queue.SayEngine}
|
||||||
|
asker := occ.EngineAsker(node)
|
||||||
|
h.Asker = &occ.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||||
|
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no
|
||||||
|
// link was open, an ask past its expiry): besides each link's opening.
|
||||||
|
const reconcileEvery = 5 * time.Minute
|
||||||
|
|
||||||
|
func reconcileActs(ctx context.Context, h *link.HandActs) {
|
||||||
|
t := time.NewTicker(reconcileEvery)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
h.Reconcile()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositionReadFor bounds one read of the verb composition: the revisions of one key from the bucket.
|
||||||
|
const compositionReadFor = 20 * time.Second
|
||||||
|
|
||||||
|
// compositionFrom answers the verb composition from the node agent that runs now (issue 774); refused in words while
|
||||||
|
// none does.
|
||||||
|
func compositionFrom(current *agentNow, withheld func(ctx context.Context, module string) ([]link.WithheldSubject, string)) func(module, digest string, diff bool) (any, error) {
|
||||||
|
return func(module, digest string, diff bool) (any, error) {
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no composition yet")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), compositionReadFor)
|
||||||
|
defer cancel()
|
||||||
|
view, err := ag.Composition(ctx, module, digest, diff)
|
||||||
|
if err != nil {
|
||||||
|
return view, err
|
||||||
|
}
|
||||||
|
// And what the grants part withholds of it here, through its verb withheld (design 46 approval mark §6).
|
||||||
|
subjects, unknown := withheld(ctx, module)
|
||||||
|
for _, s := range subjects {
|
||||||
|
view.Withheld = append(view.Withheld, agent.WithheldView{Subject: s.Subject, Why: s.Why})
|
||||||
|
}
|
||||||
|
view.WithheldUnknown = unknown
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeReadFor bounds the verb compose's wait for its pass: a pass that waits on a seat longer says what it waits on.
|
||||||
|
const composeReadFor = 2 * time.Minute
|
||||||
|
|
||||||
|
// composeFrom answers the verb compose (issue 774): a pass of the composer that runs now, and with module that module's
|
||||||
|
// apply tried again by the node agent; refused in words while the node composes nothing of its own.
|
||||||
|
func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Composer]) func(module string) (any, error) {
|
||||||
|
return func(module string) (any, error) {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (it applies the controller's " +
|
||||||
|
"compositions, or its node agent is starting), so there is no pass to make")
|
||||||
|
}
|
||||||
|
out := map[string]any{}
|
||||||
|
if module != "" {
|
||||||
|
// Only a failed apply that is not held is tried again; what runs is never taken away (review round 1: the
|
||||||
|
// verb has no warrant, ADR 0563 §2, and a hold for a failed backup is the operator's to release, ADR 0568 §4).
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so no apply is tried again")
|
||||||
|
}
|
||||||
|
if err := ag.RetryFailed(module); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["retried"] = module
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), composeReadFor)
|
||||||
|
defer cancel()
|
||||||
|
pass, err := c.ComposeNow(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["pass"] = pass
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// packagesReadFor bounds the verb packages' reads and its write.
|
||||||
|
const packagesReadFor = 20 * time.Second
|
||||||
|
|
||||||
|
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages;
|
||||||
|
// with write, the key written again from what the
|
||||||
|
// composer that runs now handed, refused in words while the node composes none of its own.
|
||||||
|
func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) {
|
||||||
|
return func(write bool) (any, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), packagesReadFor)
|
||||||
|
defer cancel()
|
||||||
|
var wrote *bool
|
||||||
|
if write {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (its node agent is starting), " +
|
||||||
|
"so it has no packages of its own to write")
|
||||||
|
}
|
||||||
|
w, err := c.WritePackages(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("this node's key of %s is not written: %w", link.PackagesBucket, err)
|
||||||
|
}
|
||||||
|
wrote = &w
|
||||||
|
}
|
||||||
|
key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node)
|
||||||
|
v := link.PackagesAnswer(node, key, keyErr)
|
||||||
|
v.Wrote = wrote
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeCandidateFrom answers the verb compose-candidate (novox/hq ADR 0593 §7): the check's candidate composed on this
|
||||||
|
// node in shadow by the run's shadow, applying and writing nothing, within the shadow's own bound (shorter than
|
||||||
|
// nec.AnswerBound, selfcompose candidateWithin). The answer is
|
||||||
|
// the seat's shape (nodeengine.ComposeCandidateAnswer), checked as it is encoded; a node whose shadow does not run yet
|
||||||
|
// answers unready, which the checker takes as no answer.
|
||||||
|
func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow]) func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
return func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
ask := nodeengine.ComposeCandidateInput{Candidate: candidate, Input: input, Call: call}
|
||||||
|
if _, err := nec.EncodeAsk(ask); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var a nodeengine.ComposeCandidateAnswer
|
||||||
|
if s := shadow.Load(); s == nil {
|
||||||
|
// Unready, never an error the checker reads words of: its agent is starting (ADR 0593 §9).
|
||||||
|
a = nodeengine.ComposeCandidateAnswer{Node: node, Candidate: candidate, Verdict: nec.Unready,
|
||||||
|
Refused: "this node's shadow does not run now (its node agent is starting), so it composes no candidate"}
|
||||||
|
} else {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), nec.AnswerBound)
|
||||||
|
defer cancel()
|
||||||
|
a = s.ComposeCandidate(ctx, ask)
|
||||||
|
}
|
||||||
|
return answerComposeCandidate(a, candidate, input, call, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstViewsFrom answers the seat's verb first-views from the composer of this run (design 47 G §698): the views the node-engine
|
||||||
|
// follows (none), the entries it keeps and the wants it composes from.
|
||||||
|
func firstViewsFrom(composer *atomic.Pointer[selfcompose.Composer]) func() (any, error) {
|
||||||
|
return func() (any, error) {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (it applies the controller's compositions, or its node agent is starting), so it keeps no entry")
|
||||||
|
}
|
||||||
|
views, entries, wants := c.FirstViews()
|
||||||
|
return map[string]any{"first-views": views, "entries": entries, "wants": wants}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// entryAgainFrom asks one entry of the catalog half again, for the seat's verb entry-again (design 47 G §698).
|
||||||
|
func entryAgainFrom(composer *atomic.Pointer[selfcompose.Composer]) func(ctx context.Context, fingerprint string) error {
|
||||||
|
return func(ctx context.Context, fingerprint string) error {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return errors.New("this node composes none of its own modules now, so it keeps no entry to ask again")
|
||||||
|
}
|
||||||
|
return c.EntryAgain(ctx, fingerprint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerComposeCandidate encodes the answer of compose-candidate. The verb is marked chunks (A-237b, A-237c): asked with input and a
|
||||||
|
// call token, the composition input goes in chunks over the link's connection, whatever its size, and the answer names them;
|
||||||
|
// asked with no token (the checker's, until G removes it, G-6 §321, A-237c) it stays inline in the answer.
|
||||||
|
func answerComposeCandidate(a nodeengine.ComposeCandidateAnswer, candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
|
||||||
|
var chunked []byte
|
||||||
|
if input && call != "" && a.Input != nil {
|
||||||
|
if line == nil {
|
||||||
|
return nil, errors.New("this engine has no bus connection to send the composition input on")
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(a.Input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the composition input of %s cannot be encoded: %w", candidate, err)
|
||||||
|
}
|
||||||
|
chunked, a.Input = body, nil
|
||||||
|
a.Chunks = int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
||||||
|
}
|
||||||
|
raw, err := nec.EncodeAnswer(a)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("this node's answer to %s does not fit the seat's shape: %w", candidate, err)
|
||||||
|
}
|
||||||
|
if chunked != nil {
|
||||||
|
send := buscalls.SeatChunks{Seat: nodeengine.Seat, Verb: nodeengine.VerbComposeCandidate, Call: call}
|
||||||
|
// The answer names the chunks first; the send runs after it and ends at a chunk the caller does not acknowledge.
|
||||||
|
go func() {
|
||||||
|
if err := send.Send(line, chunked, buscalls.SeatAckWithin, log.Printf); err != nil {
|
||||||
|
log.Printf("the send %s of the composition input of %s: %v", call, candidate, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
return json.RawMessage(raw), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// enrolThroughSeat joins this machine through the seat mesh-node-enroller (novox/hq ADR 0587 §6, issue 774): it redeems the token
|
||||||
|
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
|
||||||
|
// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this
|
||||||
|
// token (link.ErrNoSeat) is said as the error: nothing was sent or kept.
|
||||||
|
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
|
||||||
|
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error {
|
||||||
|
|
||||||
|
req := meshnodeenrollerclient.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
|
||||||
|
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
|
||||||
|
if found != nil {
|
||||||
|
var err error
|
||||||
|
if req.Tunnel, err = json.Marshal(found); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(meshnodeenrollerclient.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
|
||||||
|
to := link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}
|
||||||
|
membership := link.Membership{Node: node, Broker: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}
|
||||||
|
j := link.Joining{Node: node, Signer: token.SignerKey(),
|
||||||
|
Redeem: func(ctx context.Context) (meshnodeenrollerclient.RedeemAnswer, error) {
|
||||||
|
return link.Redeem(ctx, to, node, token.Secret, req, opts.timeout)
|
||||||
|
},
|
||||||
|
Open: func(a meshnodeenrollerclient.RedeemAnswer) (string, error) {
|
||||||
|
value, err := sealing.Open(a.Account)
|
||||||
|
return string(value), err
|
||||||
|
},
|
||||||
|
Keep: func(a meshnodeenrollerclient.RedeemAnswer, password string) error {
|
||||||
|
// The mesh's name for this node wins over what the machine called itself, as on the controller's path.
|
||||||
|
mine.Node = a.Node
|
||||||
|
mine.Membership = identity.Membership{Broker: token.Broker, Fingerprint: token.Fingerprint, Signer: token.Signer,
|
||||||
|
Password: password, Transport: link.OnNATS}
|
||||||
|
if err := writeLogins(loginsFile(opts.state), link.LoginsState{Current: a.Login, Values: map[string]string{a.Login: password}}); err != nil {
|
||||||
|
return fmt.Errorf("cannot keep the login %s: %w", a.Login, err)
|
||||||
|
}
|
||||||
|
if err := identity.Save(identityPath, mine); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !mine.Membership.Joined() {
|
||||||
|
return errors.New("the answer did not say how to reach the mesh again, so the identity could not be used after a restart")
|
||||||
|
}
|
||||||
|
return keepJoinedKeys(opts, mine, sealing, serving)
|
||||||
|
},
|
||||||
|
Dial: func(ctx context.Context, login, password string) (link.Confirming, error) {
|
||||||
|
return link.DialConfirming(ctx, membership, login, password, opts.timeout)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
answer, err := j.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("\nenrolled as %s through the seat mesh-node-enroller, dialled as %s and confirmed\n", answer.Node, answer.Login)
|
||||||
|
fmt.Printf(" identity %s\n", identityPath)
|
||||||
|
fmt.Println(" nothing is applied until the assignment of this node names a module")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
|
||||||
|
// not there looks joined and cannot come up, so the identity is written first and a failure here is said plainly.
|
||||||
|
func keepJoinedKeys(opts options, mine identity.Identity, sealing identity.SealingKey, serving identity.ServingKey) error {
|
||||||
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state), []byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's overlay key: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(identity.SealingKeyPath(opts.state), []byte(sealing.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's sealing key: %w", err)
|
||||||
|
}
|
||||||
|
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this node's serving key: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be
|
|
||||||
// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node
|
|
||||||
// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before
|
|
||||||
// the machine is read or touched, with a report naming what was refused and what is held, and the
|
|
||||||
// kept declaration left as it was.
|
|
||||||
func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
opts := options{state: filepath.Join(dir, "state.json")}
|
|
||||||
_, private, err := ed25519.GenerateKey(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
target := filepath.Join(dir, "a.conf")
|
|
||||||
declared := func(epoch, sequence int64) store.Declared {
|
|
||||||
body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence,
|
|
||||||
"resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
|
|
||||||
}
|
|
||||||
held := declared(57, 3)
|
|
||||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stale := declared(41, 12)
|
|
||||||
report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil)
|
|
||||||
if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) ||
|
|
||||||
report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) ||
|
|
||||||
report.Refused == "" {
|
|
||||||
t.Fatalf("the refusal does not name what was refused and what is held: %+v", report)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatal("the refused declaration touched the machine")
|
|
||||||
}
|
|
||||||
kept, err := store.ReadDeclared(store.DeclaredPath(opts.state))
|
|
||||||
if err != nil || string(kept.Declaration) != string(held.Declaration) {
|
|
||||||
t.Fatal("the refused declaration replaced what this node kept")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The queue's counts are kept beside the state and read back by the next host; unreadable is an
|
|
||||||
// error, never zero.
|
|
||||||
func TestTheNumbersSurviveTheHost(t *testing.T) {
|
|
||||||
state := filepath.Join(t.TempDir(), "state.json")
|
|
||||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 {
|
|
||||||
t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err)
|
|
||||||
}
|
|
||||||
if err := numbersBeside(state).Save(41, 2); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 {
|
|
||||||
t.Fatalf("read back %d, %d, %v", sequence, refused, err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, _, err := numbersBeside(state).Read(); err == nil {
|
|
||||||
t.Fatal("unreadable numbers were read as zero")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aFilesLink is a link that serves objects by name, or fails every read with err.
|
||||||
|
type aFilesLink struct {
|
||||||
|
objects map[string][]byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
|
||||||
|
if a.err != nil {
|
||||||
|
return nil, a.err
|
||||||
|
}
|
||||||
|
if b, ok := a.objects[name]; ok {
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
return nil, os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceDeclaration(path string, body []byte) []byte {
|
||||||
|
return []byte(`{"declaration":1,"resources":[{"id":"m.big","type":"file","path":"` + path + `","digest":"` +
|
||||||
|
declaration.DigestOf(body) + `","size":` + strconv.Itoa(len(body)) + `}]}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1695/AC1 and AC2, through the whole apply: a reference is fetched, verified and written; a differing object
|
||||||
|
// refuses the declaration whole and writes nothing.
|
||||||
|
func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
||||||
|
unansweringPacman(t)
|
||||||
|
body := []byte(strings.Repeat("a line of a big file\n", 4000))
|
||||||
|
name, _ := declaration.ObjectName(declaration.DigestOf(body))
|
||||||
|
dir := t.TempDir()
|
||||||
|
target := filepath.Join(dir, "big")
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||||
|
raw := referenceDeclaration(target, body)
|
||||||
|
|
||||||
|
bad := append([]byte(nil), body...)
|
||||||
|
bad[10] ^= 1
|
||||||
|
ctx := link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: bad}})
|
||||||
|
report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||||
|
if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
|
||||||
|
t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(target); err == nil {
|
||||||
|
t.Error("1695/AC2: a file was written from an object that differs")
|
||||||
|
}
|
||||||
|
if failedWith(report.Refused).Transient {
|
||||||
|
t.Errorf("an object that differs is a real failure, and was taken as the bus not answering: %s", report.Refused)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus answers no JetStream request while it reloads its accounts (2026-10-25 23:04:24, mesh-vault on the bus
|
||||||
|
// node): the failure is transient, so the agent tries it again, and the detail kept is its cause, the tail.
|
||||||
|
noResponders := fmt.Errorf("fetching the object 2bd98ebd of this node's own object bucket: reading the chunks of "+
|
||||||
|
"object %s: %w", strings.Repeat("x", 300), nats.ErrNoResponders)
|
||||||
|
ctx = link.WithFiles(context.Background(), aFilesLink{err: noResponders})
|
||||||
|
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||||
|
failed := failedWith("a file of this composition could not be had by its digest: file mesh-vault.kept: " + report.Refused)
|
||||||
|
if !failed.Transient || !strings.HasSuffix(failed.Detail, nats.ErrNoResponders.Error()) || len(failed.Detail) > 310 ||
|
||||||
|
!strings.HasPrefix(failed.Detail, "a file of this composition") {
|
||||||
|
t.Errorf("a fetch that met no responders: transient %v, detail %q (%d bytes); want transient, its head and its cause",
|
||||||
|
failed.Transient, failed.Detail, len(failed.Detail))
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: body}})
|
||||||
|
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||||
|
if report.Refused != "" {
|
||||||
|
t.Fatalf("1695/AC1: refused: %s", report.Refused)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(target)
|
||||||
|
if err != nil || string(got) != string(body) {
|
||||||
|
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
|
||||||
|
// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
|
||||||
|
gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
|
||||||
|
never := []byte(strings.Repeat("never had here\n", 6000))
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
raw []byte
|
||||||
|
onDisk []byte // nil removes the file
|
||||||
|
refused string // "" applies
|
||||||
|
}{
|
||||||
|
{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
|
||||||
|
{"the file here is gone, its object deleted", raw, nil, ""},
|
||||||
|
{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
|
||||||
|
} {
|
||||||
|
if row.onDisk == nil {
|
||||||
|
_ = os.Remove(target)
|
||||||
|
} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
|
||||||
|
if row.refused == "" {
|
||||||
|
got, _ := os.ReadFile(target)
|
||||||
|
if report.Refused != "" || string(got) != string(body) {
|
||||||
|
t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
|
||||||
|
row.name, report.Refused, len(got))
|
||||||
|
}
|
||||||
|
} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
|
||||||
|
t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago,
|
||||||
|
// while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with
|
||||||
|
// no object; a file someone else changed since is left as named.
|
||||||
|
newer := []byte(strings.Repeat("the export the node wrote since\n", 3000))
|
||||||
|
newerName, _ := declaration.ObjectName(declaration.DigestOf(newer))
|
||||||
|
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}})
|
||||||
|
if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Fatalf("924: the newer export: %s", report.Refused)
|
||||||
|
}
|
||||||
|
stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) +
|
||||||
|
`","size":` + strconv.Itoa(len(body)) + `}`)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
edit []byte // what someone else wrote there since; nil for nobody
|
||||||
|
digest string
|
||||||
|
}{
|
||||||
|
{"the file is what the node wrote", nil, declaration.DigestOf(newer)},
|
||||||
|
{"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)},
|
||||||
|
} {
|
||||||
|
if row.edit != nil {
|
||||||
|
if err := os.WriteFile(target, row.edit, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runs := runsAsWritten([]engcomp.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state)
|
||||||
|
var got struct {
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(runs[0].Resources[0], &got)
|
||||||
|
if got.Digest != row.digest {
|
||||||
|
t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest)
|
||||||
|
}
|
||||||
|
if row.edit == nil {
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`)
|
||||||
|
if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
||||||
|
// as it is, any other refuses and writes nothing — never an empty file.
|
||||||
|
func TestTheHandApplyNeverWritesAReferenceEmpty(t *testing.T) {
|
||||||
|
unansweringPacman(t)
|
||||||
|
body := []byte(strings.Repeat("a line of a big file\n", 4000))
|
||||||
|
dir := t.TempDir()
|
||||||
|
target := filepath.Join(dir, "big")
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||||
|
raw := referenceDeclaration(target, body)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := runApply(context.Background(), opts, d, raw, fromFile); err == nil || !strings.Contains(err.Error(), "digest") {
|
||||||
|
t.Errorf("1695/AC8: a reference with no file and no bus was not refused: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(target); err == nil {
|
||||||
|
t.Error("1695/AC8: a file was written for a reference that could not be had")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(target, body, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, _ = declaration.ParseFileTrusted(raw)
|
||||||
|
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil && !strings.Contains(err.Error(), "applied only in part") {
|
||||||
|
t.Fatalf("1695/AC8: %v", err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(target); string(got) != string(body) {
|
||||||
|
t.Errorf("1695/AC8: the file holds %d bytes, want %d", len(got), len(body))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
||||||
|
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
||||||
|
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) {
|
||||||
|
e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say)
|
||||||
|
fillingNow.Store(e)
|
||||||
|
defer fillingNow.CompareAndSwap(e, nil)
|
||||||
|
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
||||||
|
say("filled files are not written on request on this machine: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fillingNow is the filling this engine serves, for knockFilling; nil while none is served.
|
||||||
|
var fillingNow atomic.Pointer[filling.Engine]
|
||||||
|
|
||||||
|
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
||||||
|
// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled
|
||||||
|
// (agent.Env.Settled, "<module> settled"), the haves were read at the start, or a module was taken as running
|
||||||
|
// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said.
|
||||||
|
func knockFilling(after string, say func(string)) {
|
||||||
|
e := fillingNow.Load()
|
||||||
|
if e == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
told, err := e.Knock(filling.Socket)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
say(fmt.Sprintf("after %s: %v", after, err))
|
||||||
|
case told != "":
|
||||||
|
say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
||||||
|
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
||||||
|
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
||||||
|
//
|
||||||
|
// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a
|
||||||
|
// file so filled is written 0600 when the declaration gives no mode.
|
||||||
|
func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine {
|
||||||
|
return &filling.Engine{
|
||||||
|
Declared: func() (*declaration.Declaration, error) {
|
||||||
|
return declaredNow(opts.state, signer)
|
||||||
|
},
|
||||||
|
UIDOf: filling.UIDOf,
|
||||||
|
Lock: func() (func(), error) {
|
||||||
|
applying.Lock()
|
||||||
|
unlock, err := store.Lock(opts.state, nil)
|
||||||
|
if err != nil {
|
||||||
|
applying.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return func() { unlock(); applying.Unlock() }, nil
|
||||||
|
},
|
||||||
|
Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) {
|
||||||
|
built, err := system.For(builtFor)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
content, secret, err := secrets.fill(ctx, filledModule(d, path), content)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("%s was not written: %w", path, err)
|
||||||
|
}
|
||||||
|
report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner,
|
||||||
|
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)),
|
||||||
|
apply.Filled{Secret: secret})
|
||||||
|
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||||
|
if err := store.Save(opts.state, updated); err != nil {
|
||||||
|
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
||||||
|
err.Error()))
|
||||||
|
}
|
||||||
|
action, digest := "", ""
|
||||||
|
for _, o := range report.Outcomes {
|
||||||
|
if o.Target == path && o.Type == string(declaration.TypeFile) {
|
||||||
|
action = o.Action
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rec, ok := updated.At(string(declaration.TypeFile), path); ok {
|
||||||
|
digest = rec.Wrote
|
||||||
|
}
|
||||||
|
return action, digest, writeErr
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// filledModule is the module whose file the declaration declares at path, or "" when none or several do.
|
||||||
|
func filledModule(d *declaration.Declaration, path string) string {
|
||||||
|
module := ""
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if f, ok := r.(*declaration.File); ok && f.Path == path {
|
||||||
|
if module != "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
module, _ = d.ModuleOfID(f.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return module
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key.
|
||||||
|
func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets {
|
||||||
|
p := provisionSecrets{node: node, open: sealOpener(statePath)}
|
||||||
|
if queue == nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
// What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key.
|
||||||
|
p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
wctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
updates, err := queue.KVWatch(wctx, buckets.Provisions, prefix+">")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case u, ok := <-updates:
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("the watch of %s ended before it caught up", buckets.Provisions)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case u.Caught:
|
||||||
|
return out, nil
|
||||||
|
case u.Deleted:
|
||||||
|
delete(out, u.Key)
|
||||||
|
default:
|
||||||
|
out[u.Key] = u.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
return queue.AskVault(ctx, "own", args)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
|
||||||
|
// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
|
||||||
|
// node's state as an apply's outcome is.
|
||||||
|
func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
me, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if me.Uid == "0" {
|
||||||
|
t.Skip("run as root, whose account is never the provisioner's")
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
|
public, private, err := ed25519.GenerateKey(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(dir, "m.conf")
|
||||||
|
raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
|
||||||
|
"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
|
||||||
|
"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(opts.state),
|
||||||
|
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
|
||||||
|
// the opener reads the box's "v").
|
||||||
|
provision := map[string]string{}
|
||||||
|
vaultName := "provision/n2/db"
|
||||||
|
secrets := provisionSecrets{node: "n1",
|
||||||
|
list: func(_ context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for k, v := range provision {
|
||||||
|
if strings.HasPrefix(k, prefix) {
|
||||||
|
out[k] = []byte(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
},
|
||||||
|
own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
raw, _ := json.Marshal(args)
|
||||||
|
var a struct{ Credentials []map[string]any }
|
||||||
|
_ = json.Unmarshal(raw, &a)
|
||||||
|
c := a.Credentials[0]
|
||||||
|
c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"}
|
||||||
|
return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
|
||||||
|
},
|
||||||
|
open: func(sealed string) ([]byte, error) {
|
||||||
|
var b struct{ V string }
|
||||||
|
err := json.Unmarshal([]byte(sealed), &b)
|
||||||
|
return []byte(b.V), err
|
||||||
|
}}
|
||||||
|
e := fillingEngine(opts, public, secrets, func(string) {})
|
||||||
|
uid := uint32(os.Getuid())
|
||||||
|
if err := e.Accepts(uid); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An apply holds the lock.
|
||||||
|
applying.Lock()
|
||||||
|
done := make(chan filling.Answer)
|
||||||
|
go func() {
|
||||||
|
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case a := <-done:
|
||||||
|
applying.Unlock()
|
||||||
|
t.Fatalf("written while an apply held the lock: %+v", a)
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
applying.Unlock()
|
||||||
|
t.Fatal("the file was written while an apply held the lock")
|
||||||
|
}
|
||||||
|
applying.Unlock()
|
||||||
|
a := <-done
|
||||||
|
if a.Refused != "" || a.Failed != "" || a.Action != "created" {
|
||||||
|
t.Fatalf("the write after the apply: %+v", a)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(path)
|
||||||
|
if err != nil || string(got) != "filled\n" {
|
||||||
|
t.Fatalf("the file holds %q (%v)", got, err)
|
||||||
|
}
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
|
||||||
|
t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
|
||||||
|
unlock, err := store.Lock(opts.state, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case a := <-done:
|
||||||
|
unlock()
|
||||||
|
t.Fatalf("written while another process held the apply's lock: %+v", a)
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
unlock()
|
||||||
|
if a := <-done; a.Action != "updated" {
|
||||||
|
t.Fatalf("the second write: %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
|
||||||
|
// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
|
||||||
|
keyed := func(state string, ref bool, id, local string) string {
|
||||||
|
p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"},
|
||||||
|
Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
|
||||||
|
if state != provisions.Provided {
|
||||||
|
p.Why = "the provider has not answered"
|
||||||
|
}
|
||||||
|
if ref {
|
||||||
|
p.Reference = &provisions.Reference{ID: id, Generation: 2}
|
||||||
|
}
|
||||||
|
raw, err := provisions.Encode(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, state string
|
||||||
|
ref bool
|
||||||
|
content string
|
||||||
|
want string // the file after; "" keeps what was there
|
||||||
|
mode os.FileMode
|
||||||
|
vault string // the vault's credential name, when not the one the provision names
|
||||||
|
refID string // the reference's id, when not the provision's shared value
|
||||||
|
local string // the provision's local name (ADR 0094): the placeholder names it
|
||||||
|
}{
|
||||||
|
{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
|
||||||
|
{name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true,
|
||||||
|
refID: "provision/n9/db", content: "password=${secret:db}\n"},
|
||||||
|
{name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"},
|
||||||
|
{name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true,
|
||||||
|
vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
|
||||||
|
content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
|
||||||
|
{name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin",
|
||||||
|
content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600},
|
||||||
|
{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
|
||||||
|
} {
|
||||||
|
before, _ := os.ReadFile(path)
|
||||||
|
if tc.state != "" {
|
||||||
|
id := "provision/n2/db"
|
||||||
|
if tc.refID != "" {
|
||||||
|
id = tc.refID
|
||||||
|
}
|
||||||
|
clear(provision)
|
||||||
|
key := "n1.m.db"
|
||||||
|
if tc.local != "" {
|
||||||
|
key += "." + tc.local
|
||||||
|
}
|
||||||
|
provision[key] = keyed(tc.state, tc.ref, id, tc.local)
|
||||||
|
}
|
||||||
|
vaultName = "provision/n2/db"
|
||||||
|
if tc.vault != "" {
|
||||||
|
vaultName = tc.vault
|
||||||
|
}
|
||||||
|
a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
|
||||||
|
got, _ := os.ReadFile(path)
|
||||||
|
if tc.want == "" {
|
||||||
|
if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, _ := os.Stat(path)
|
||||||
|
if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A directory used as found is never quiet (novox/hq issue 339).
|
||||||
|
//
|
||||||
|
// The engine leaves the owner and mode of a directory it did not make: changing them would be root handing a
|
||||||
|
// directory it never made to whichever account a declaration named. That is safe and, said nowhere, a module
|
||||||
|
// whose data directory it cannot write would simply fail. So each such directory is said three ways: a WARNING
|
||||||
|
// line in every apply's log; a resource of its module of kind `directory`, stated unhealthy on every look,
|
||||||
|
// which the controller raises as that module's condition naming the directory and its owner; and a way out,
|
||||||
|
// which is the operator's line at the controller's terminal — `nox node hand-over <node> <path>` on the
|
||||||
|
// control-node (novox/hq ADR 0272, issue 356) — after which the next apply gives it the declared owner and
|
||||||
|
// mode like any directory of the mesh's. The controller carries that line to this engine over the bus
|
||||||
|
// (link.AskHandOverSubject); the engine's own `hand-over` subcommand is the same act for a machine with no
|
||||||
|
// mesh to reach.
|
||||||
|
|
||||||
|
// foundState is where the node's state is, for the looks to read the directories used as found from; empty
|
||||||
|
// until the engine runs.
|
||||||
|
var foundState atomic.Value
|
||||||
|
|
||||||
|
// foundNode is this node's name in the mesh, for the way out to name it; empty until the engine is enrolled.
|
||||||
|
var foundNode atomic.Value
|
||||||
|
|
||||||
|
// handedOverFrom has the apply read the directories handed over from beside the state at statePath.
|
||||||
|
func handedOverFrom(statePath string) {
|
||||||
|
apply.HandedOver = func() map[string]bool {
|
||||||
|
all, err := store.LoadHandedOver(statePath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: %v; no directory counts as handed over until it reads\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, h := range all {
|
||||||
|
out[apply.HandOverKey(h.ID, h.Path)] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// foundDirectories is every directory the node's state records as used as found, sorted by id.
|
||||||
|
func foundDirectories(known store.State) []store.Applied {
|
||||||
|
var out []store.Applied
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if r.Type == "directory" && r.AsFound {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverLine is the way out of a directory used as found, with this node's name when it is known: the node seat's
|
||||||
|
// hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5). The controller
|
||||||
|
// that answered `nox node hand-over` ended by its unassignment (issue 1126), so that line has no answerer. The path is
|
||||||
|
// the resource's target beside the reason, never in it: the operator's channel withholds a path (ADR 0234 §6).
|
||||||
|
func handOverLine() string {
|
||||||
|
node, _ := foundNode.Load().(string)
|
||||||
|
if node == "" {
|
||||||
|
node = "<node>"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
||||||
|
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
|
||||||
|
// that hands it over. The path is the target, which the controller keeps as evidence.
|
||||||
|
func withFoundDirectories(h *link.Health, found []store.Applied) *link.Health {
|
||||||
|
for _, r := range found {
|
||||||
|
at := strings.LastIndex(r.ID, ".")
|
||||||
|
if at <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
owner, mode := "unknown", "unknown"
|
||||||
|
if info, err := os.Lstat(r.Target); err == nil {
|
||||||
|
mode = fmt.Sprintf("%o", info.Mode().Perm())
|
||||||
|
if uid, gid, ok := apply.OwnerOf(info); ok {
|
||||||
|
owner = fmt.Sprintf("%d:%d", uid, gid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
declared := r.Declared
|
||||||
|
if declared == "" {
|
||||||
|
declared = "another owner or mode"
|
||||||
|
}
|
||||||
|
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.ID[:at], Resource: r.ID,
|
||||||
|
Kind: link.KindDirectory, Target: r.Target, State: link.StateUnhealthy,
|
||||||
|
Reason: fmt.Sprintf("%s owned by %s, mode %s, as found; %s was declared and not given it — %s, with its "+
|
||||||
|
"path, hands it to the mesh", link.ReasonUsedAsFound, owner, mode, declared, handOverLine()),
|
||||||
|
Since: foundSince(r).UTC()})
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// foundNow is the directories used as found, read from the node's state; none when it cannot be read, which
|
||||||
|
// the apply says on its own.
|
||||||
|
func foundNow() []store.Applied {
|
||||||
|
path, _ := foundState.Load().(string)
|
||||||
|
if path == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
known, err := store.Load(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return foundDirectories(known)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverFound hands one directory used as found to the mesh, on the operator's word carried by the
|
||||||
|
// controller (link.HandOverFunc) or typed at this machine. It changes nothing on disk itself; the next apply
|
||||||
|
// gives the directory its declared owner and mode, and says so.
|
||||||
|
//
|
||||||
|
// **Every value is judged, and a refusal records nothing.** The path is the directory's absolute path exactly
|
||||||
|
// as the engine states it — clean, so `..` and a doubled separator never name a directory; one this machine's
|
||||||
|
// state records as used as found; on disk a directory and not a link, reached through no link. The engine
|
||||||
|
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
||||||
|
// apply, and a path that resolves elsewhere would give that elsewhere.
|
||||||
|
func handOverFound(statePath, path, by string) (string, error) {
|
||||||
|
found, _, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
||||||
|
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at
|
||||||
|
// that path, as the engine states it, and what is on disk there.
|
||||||
|
func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) {
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
||||||
|
}
|
||||||
|
if filepath.Clean(path) != path {
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
||||||
|
"separator); nothing was handed over", path)
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
return store.Applied{}, nil, err
|
||||||
|
}
|
||||||
|
var found *store.Applied
|
||||||
|
for _, r := range foundDirectories(known) {
|
||||||
|
if filepath.Clean(r.Target) == path {
|
||||||
|
r := r
|
||||||
|
found = &r
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil {
|
||||||
|
return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
||||||
|
"the ones that are); nothing was handed over")
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
||||||
|
case info.Mode()&os.ModeSymlink != 0:
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
||||||
|
case !info.IsDir():
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
||||||
|
}
|
||||||
|
resolved, err := filepath.EvalSymlinks(path)
|
||||||
|
if err != nil {
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
||||||
|
}
|
||||||
|
if resolved != path {
|
||||||
|
return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
||||||
|
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
||||||
|
}
|
||||||
|
return *found, info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
||||||
|
func handOverFrom(statePath string) link.HandOverFunc {
|
||||||
|
return func(path, by string) (string, error) { return handOverFound(statePath, path, by) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverCommand is `nox-mesh-host hand-over <dir>` at this machine, as root: the same act, for a machine
|
||||||
|
// with no mesh to reach. The operator's line is `nox node hand-over <node> <dir>` on the control-node.
|
||||||
|
func handOverCommand(opts options) error {
|
||||||
|
by := "a person at the machine"
|
||||||
|
if u, err := user.Current(); err == nil {
|
||||||
|
by = u.Username + " at the machine"
|
||||||
|
}
|
||||||
|
said, err := handOverFound(opts.state, filepath.Clean(opts.file), by)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintln(opts.out, said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// foundSince is when a directory was first found as it is: kept across applies, so the controller can tell one
|
||||||
|
// a send found from one found before it. A record from before that was kept says when it was last applied.
|
||||||
|
func foundSince(r store.Applied) time.Time {
|
||||||
|
if !r.FoundAt.IsZero() {
|
||||||
|
return r.FoundAt
|
||||||
|
}
|
||||||
|
return r.AppliedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
func orUnsaid(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "the declared owner and mode"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every directory used as found is its module's resource of kind directory, unhealthy, naming its owner, on
|
||||||
|
// every statement (novox/hq issue 339): the controller raises that as the module's condition. The way out it
|
||||||
|
// names is the node seat's hand-over, with this node's name (novox/hq ADR 0563 §5).
|
||||||
|
func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
known := store.State{Resources: []store.Applied{
|
||||||
|
{ID: "notes.data", Type: "directory", Target: dir, AsFound: true, Declared: "root, mode 755", AppliedAt: time.Now()},
|
||||||
|
{ID: "notes.cache", Type: "directory", Target: dir},
|
||||||
|
}}
|
||||||
|
foundNode.Store("laptop")
|
||||||
|
t.Cleanup(func() { foundNode.Store("") })
|
||||||
|
h := withFoundDirectories(&link.Health{}, foundDirectories(known))
|
||||||
|
if len(h.Resources) != 1 {
|
||||||
|
t.Fatalf("stated: %+v", h.Resources)
|
||||||
|
}
|
||||||
|
r := h.Resources[0]
|
||||||
|
if r.Module != "notes" || r.Kind != link.KindDirectory || r.State != link.StateUnhealthy || r.Target != dir ||
|
||||||
|
!strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) || !strings.Contains(r.Reason, "owned by ") ||
|
||||||
|
!strings.Contains(r.Reason, "root, mode 755") {
|
||||||
|
t.Fatalf("stated as: %+v", r)
|
||||||
|
}
|
||||||
|
// The path is the target, kept as evidence, and never in the reason: the operator's channel withholds a path
|
||||||
|
// (novox/hq ADR 0234 §6). The way out is the node seat's hand-over (ADR 0563 §5), never the controller's terminal line,
|
||||||
|
// which nothing answers since the controller ended (issue 1126).
|
||||||
|
if !strings.Contains(r.Reason, "`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") ||
|
||||||
|
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
||||||
|
t.Fatalf("the way out is not the node seat's hand-over, or carries the path: %q", r.Reason)
|
||||||
|
}
|
||||||
|
foundNode.Store("")
|
||||||
|
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||||
|
"`<node>/node-engine.hand-over`") {
|
||||||
|
t.Fatalf("before enrolment the line names a placeholder: %q", r.Reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// realDir is a temporary directory with every link in its path resolved, so a judgement of links is about the
|
||||||
|
// links the test makes.
|
||||||
|
func realDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// A hand-over records a directory used as found, and refuses one that is not — through the engine's own
|
||||||
|
// subcommand and through the controller's ask alike, which are one judgement.
|
||||||
|
func TestHandOverRecordsOnlyAFoundDirectory(t *testing.T) {
|
||||||
|
dir := realDir(t)
|
||||||
|
statePath := filepath.Join(dir, "state.json")
|
||||||
|
found := filepath.Join(dir, "found")
|
||||||
|
if err := os.Mkdir(found, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(filepath.Join(dir, "mine"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.Save(statePath, store.State{Resources: []store.Applied{
|
||||||
|
{ID: "notes.data", Type: "directory", Target: found, AsFound: true, Declared: "root, mode 755"},
|
||||||
|
{ID: "notes.mine", Type: "directory", Target: filepath.Join(dir, "mine")},
|
||||||
|
}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := handOverCommand(options{state: statePath, file: filepath.Join(dir, "mine"), out: &out}); err == nil {
|
||||||
|
t.Fatal("a directory the mesh made was handed over")
|
||||||
|
}
|
||||||
|
if err := handOverCommand(options{state: statePath, file: found + "/", out: &out}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
all, err := store.LoadHandedOver(statePath)
|
||||||
|
if err != nil || len(all) != 1 || all[0].Path != found || !strings.HasSuffix(all[0].By, " at the machine") {
|
||||||
|
t.Fatalf("recorded: %+v %v", all, err)
|
||||||
|
}
|
||||||
|
handedOverFrom(statePath)
|
||||||
|
if !apply.HandedOver()[apply.HandOverKey("notes.data", found)] {
|
||||||
|
t.Fatal("the apply does not see the hand-over")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "root, mode 755") {
|
||||||
|
t.Fatalf("said: %s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller's ask is the same judgement, recording who asked in the controller's words.
|
||||||
|
_ = os.Remove(store.HandedOverPath(statePath))
|
||||||
|
said, err := handOverFrom(statePath)(found, "jo through mesh-cli on anchor")
|
||||||
|
if err != nil || !strings.Contains(said, "by jo through mesh-cli on anchor") || !strings.Contains(said, "root, mode 755") {
|
||||||
|
t.Fatalf("the ask answered %q, %v", said, err)
|
||||||
|
}
|
||||||
|
all, _ = store.LoadHandedOver(statePath)
|
||||||
|
if len(all) != 1 || all[0].By != "jo through mesh-cli on anchor" || all[0].ID != "notes.data" {
|
||||||
|
t.Fatalf("recorded: %+v", all)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every value of a hand-over is judged before anything is recorded, and each refusal records nothing: a relative
|
||||||
|
// path, a path that is not the engine's statement of the directory (`..`, a doubled or trailing separator), one
|
||||||
|
// not used as found, a link at the path, a link in the path, a file, a directory that is gone.
|
||||||
|
func TestAHandOverIsRefusedForEveryPathThatIsNotTheFoundDirectory(t *testing.T) {
|
||||||
|
dir := realDir(t)
|
||||||
|
statePath := filepath.Join(dir, "state.json")
|
||||||
|
found := filepath.Join(dir, "found")
|
||||||
|
if err := os.Mkdir(found, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A link at the path of a second found directory, and a found directory below a linked directory.
|
||||||
|
linked := filepath.Join(dir, "linked")
|
||||||
|
if err := os.Symlink(found, linked); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
through := filepath.Join(dir, "via", "data")
|
||||||
|
if err := os.Symlink(dir, filepath.Join(dir, "via")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(filepath.Join(dir, "data"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
file := filepath.Join(dir, "file")
|
||||||
|
if err := os.WriteFile(file, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gone := filepath.Join(dir, "gone")
|
||||||
|
if err := store.Save(statePath, store.State{Resources: []store.Applied{
|
||||||
|
{ID: "notes.data", Type: "directory", Target: found, AsFound: true},
|
||||||
|
{ID: "notes.linked", Type: "directory", Target: linked, AsFound: true},
|
||||||
|
{ID: "notes.through", Type: "directory", Target: through, AsFound: true},
|
||||||
|
{ID: "notes.file", Type: "directory", Target: file, AsFound: true},
|
||||||
|
{ID: "notes.gone", Type: "directory", Target: gone, AsFound: true},
|
||||||
|
}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Each refused by its own check, which the reason names.
|
||||||
|
refused := map[string][2]string{
|
||||||
|
"a relative path": {"found", "not one"},
|
||||||
|
"a path with ..": {dir + "/x/../found", "as the engine states it"},
|
||||||
|
"a doubled separator": {dir + "//found", "as the engine states it"},
|
||||||
|
"a trailing separator": {found + "/", "as the engine states it"},
|
||||||
|
"a directory not used as found": {filepath.Join(dir, "other"), "not a directory this machine uses as found"},
|
||||||
|
"a link at the path": {linked, "is a link"},
|
||||||
|
"a link in the path": {through, "reached through a link"},
|
||||||
|
"a file": {file, "is not a directory on this machine"},
|
||||||
|
"a directory that is gone": {gone, "cannot be read on this machine"},
|
||||||
|
}
|
||||||
|
for name, c := range refused {
|
||||||
|
said, err := handOverFound(statePath, c[0], "jo")
|
||||||
|
if err == nil || said != "" {
|
||||||
|
t.Fatalf("%s (%s) was handed over: %q", name, c[0], said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "nothing was handed over") || !strings.Contains(err.Error(), c[1]) {
|
||||||
|
t.Fatalf("%s's refusal does not say %q and that nothing was handed over: %v", name, c[1], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if all, err := store.LoadHandedOver(statePath); err != nil || len(all) != 0 {
|
||||||
|
t.Fatalf("a refusal recorded something: %+v %v", all, err)
|
||||||
|
}
|
||||||
|
// The found directory itself, stated exactly, is taken.
|
||||||
|
if _, err := handOverFound(statePath, found, "jo"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if all, _ := store.LoadHandedOver(statePath); len(all) != 1 || all[0].Path != found {
|
||||||
|
t.Fatalf("recorded: %+v", all)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// owedBus is a link that takes reports and statements and keeps a have bucket in memory.
|
||||||
|
type owedBus struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
kv map[string][]byte
|
||||||
|
rv map[string]uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
||||||
|
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
||||||
|
|
||||||
|
func (b *owedBus) Get(_ context.Context, key string) ([]byte, uint64, error) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
v, ok := b.kv[key]
|
||||||
|
if !ok {
|
||||||
|
return nil, 0, link.ErrHaveAbsent
|
||||||
|
}
|
||||||
|
return v, b.rv[key], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *owedBus) Set(_ context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
if b.rv[key] != revision {
|
||||||
|
return errors.New("wrong last sequence")
|
||||||
|
}
|
||||||
|
b.kv[key], b.rv[key] = value, revision+1
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *owedBus) have(key string) (nodeengine.HaveByNodeModuleValue, bool) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
h, err := nodeengineclient.DecodeHave(b.kv[key])
|
||||||
|
return h, err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module with nothing long-running (xorg) is ok within one look of its apply, not at the five-minute statement: the
|
||||||
|
// facts look inside the apply runs while it is applying and judges nothing, and the running write then owes a look,
|
||||||
|
// which the health loop makes at once (review of fix 15, B1).
|
||||||
|
func TestAModuleWithNothingLongRunningIsOkWithinALookOfItsApply(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
bus := &owedBus{kv: map[string][]byte{}, rv: map[string]uint64{}}
|
||||||
|
queue := &link.Session{Membership: link.Membership{Node: "n1"}, Timeout: time.Second}
|
||||||
|
queue.Attach(ctx, bus)
|
||||||
|
j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fact := link.ModuleFact{Module: "xorg", FP: "src2:" + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("b", 64)}
|
||||||
|
applying := fact
|
||||||
|
applying.Phase = string(nodeengineclient.PhaseApplying)
|
||||||
|
queue.NoteModule(ctx, applying)
|
||||||
|
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil, nil)
|
||||||
|
// The facts look inside the apply, while the module is still applying.
|
||||||
|
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
||||||
|
if h, _ := bus.have("n1.xorg"); h.Health != nodeengineclient.HealthUnknown {
|
||||||
|
t.Fatalf("judged while applying: %+v", h)
|
||||||
|
}
|
||||||
|
queue.NoteModule(ctx, fact)
|
||||||
|
deadline := time.Now().Add(2 * time.Second) // well within one look (liveness.LookEvery, 15 s)
|
||||||
|
for {
|
||||||
|
h, _ := bus.have("n1.xorg")
|
||||||
|
if h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("not ok within a look of its apply: %+v", h)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldBy is the containers an open maintenance window holds still, by runtime name (ADR 0189), at now.
|
||||||
|
func heldBy(windows *apply.Windows) func(time.Time) map[string]bool {
|
||||||
|
return func(now time.Time) map[string]bool {
|
||||||
|
held := map[string]bool{}
|
||||||
|
for _, w := range windows.Open(now) {
|
||||||
|
for _, name := range w.Holds {
|
||||||
|
held[name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inactiveIn is what the agent asks at its start of a module it believes runs (ADR 0568 §1): whether the machine shows a
|
||||||
|
// long-running resource the composition declares running not active, read from the service manager and the container
|
||||||
|
// runtime and from no record. A resource held by an open maintenance window, or waiting for a library, is not counted;
|
||||||
|
// one that cannot be read is not counted either (unknown is not inactive), and neither is one the manager is restarting.
|
||||||
|
func inactiveIn(rt liveness.Runtime, held func(time.Time) map[string]bool) func(context.Context, engcomp.Composed) bool {
|
||||||
|
return func(ctx context.Context, c engcomp.Composed) bool {
|
||||||
|
d, err := declarationOf(c)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var holding map[string]bool
|
||||||
|
if held != nil {
|
||||||
|
holding = held(time.Now())
|
||||||
|
}
|
||||||
|
return anyInactive(ctx, rt, liveness.LongRunning(d, withWaiting(nil, apply.WaitingOnLibrary(d))), holding)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// anyInactive reads each group of resources once (the containers, each service manager's units) and says whether any is
|
||||||
|
// not active.
|
||||||
|
func anyInactive(ctx context.Context, rt liveness.Runtime, resources []liveness.Resource, held map[string]bool) bool {
|
||||||
|
type group struct {
|
||||||
|
kind, scope, user string
|
||||||
|
targets []string
|
||||||
|
}
|
||||||
|
var groups []*group
|
||||||
|
at := map[string]*group{}
|
||||||
|
for _, r := range resources {
|
||||||
|
if r.Kind == liveness.KindContainer && held[r.Target] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g := &group{kind: r.Kind, scope: r.Scope, user: r.User}
|
||||||
|
if r.Kind == liveness.KindContainer {
|
||||||
|
g.scope, g.user = "", ""
|
||||||
|
}
|
||||||
|
key := g.kind + "/" + g.scope + "/" + g.user
|
||||||
|
if r.Kind != liveness.KindContainer {
|
||||||
|
key = "units/" + g.scope + "/" + g.user
|
||||||
|
}
|
||||||
|
if at[key] == nil {
|
||||||
|
at[key] = g
|
||||||
|
groups = append(groups, g)
|
||||||
|
}
|
||||||
|
at[key].targets = append(at[key].targets, r.Target)
|
||||||
|
}
|
||||||
|
for _, g := range groups {
|
||||||
|
var got map[string]liveness.Observed
|
||||||
|
var err error
|
||||||
|
if g.kind == liveness.KindContainer {
|
||||||
|
got, err = rt.Containers(ctx, g.targets)
|
||||||
|
} else {
|
||||||
|
got, err = rt.Units(ctx, g.scope, g.user, g.targets)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, t := range g.targets {
|
||||||
|
if o := got[t]; !o.Found || !(o.Running || o.Restarting) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1957/AC2 and AC4 (unit; ADR 0568 §2, §4): the judge's look acts on a resource that goes down, once per edge. A repeated
|
||||||
|
// statement acts nothing, the edge clears when the resource is healthy again, and a resource held, restarting, starting or
|
||||||
|
// missing (the other path's) is no edge.
|
||||||
|
func testOnlyAResourceNewlyDownIsAnEdge(t *testing.T) {
|
||||||
|
state := func(id, s, reason string, missing bool) liveness.State {
|
||||||
|
return liveness.State{Resource: liveness.Resource{Module: "dnsmasq", ID: id}, State: s, Reason: reason, Missing: missing}
|
||||||
|
}
|
||||||
|
down := state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, false)
|
||||||
|
steps := []struct {
|
||||||
|
name string
|
||||||
|
st []liveness.State
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"down: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||||
|
{"said again while down: none", []liveness.State{down}, nil},
|
||||||
|
{"healthy again: clears", []liveness.State{state("dnsmasq.unit", liveness.Healthy, "", false)}, nil},
|
||||||
|
{"down again: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||||
|
{"held by a window: none", []liveness.State{state("dnsmasq.unit", liveness.Held, "", false)}, nil},
|
||||||
|
{"restarting: none", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonRestarting, false)}, nil},
|
||||||
|
{"starting: none", []liveness.State{state("dnsmasq.unit", liveness.Starting, "", false)}, nil},
|
||||||
|
{"missing: the other path's", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, true)}, nil},
|
||||||
|
}
|
||||||
|
var last map[string]bool
|
||||||
|
for _, s := range steps {
|
||||||
|
var got []string
|
||||||
|
got, last = newlyInactive(last, liveness.Statement{Resources: s.st})
|
||||||
|
if len(got) != len(s.want) || (len(got) == 1 && got[0] != s.want[0]) {
|
||||||
|
t.Fatalf("%s: acted on %v, want %v", s.name, got, s.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeRuntime struct {
|
||||||
|
units map[string]liveness.Observed
|
||||||
|
containers map[string]liveness.Observed
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeRuntime) Containers(_ context.Context, _ []string) (map[string]liveness.Observed, error) {
|
||||||
|
return f.containers, f.err
|
||||||
|
}
|
||||||
|
func (f fakeRuntime) Units(_ context.Context, _, _ string, _ []string) (map[string]liveness.Observed, error) {
|
||||||
|
return f.units, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1957/AC1 (unit; ADR 0568 §1): the machine's facts say a declared-running resource is not active when its unit is stopped
|
||||||
|
// or unknown to the manager; everything up, a restart in progress, a read that failed and a container held by a
|
||||||
|
// maintenance window are not.
|
||||||
|
func testTheMachineShowsAResourceNotActive(t *testing.T) {
|
||||||
|
svc := []byte(`{"id":"dnsmasq.unit","type":"service","unit":"dnsmasq.service","state":"running","module":"dnsmasq"}`)
|
||||||
|
ctr := []byte(`{"id":"dnsmasq.box","type":"container","name":"box","image":"x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","module":"dnsmasq"}`)
|
||||||
|
up := liveness.Observed{Found: true, Running: true}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
rt fakeRuntime
|
||||||
|
held map[string]bool
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"all up", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||||
|
{"unit inactive", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||||
|
{"unit unknown to the manager", fakeRuntime{containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||||
|
{"unit restarting", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true, Restarting: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||||
|
{"container not running", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, nil, true},
|
||||||
|
{"container held by a window", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, map[string]bool{"box": true}, false},
|
||||||
|
{"unread", fakeRuntime{err: errors.New("the service manager could not be read")}, nil, false},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
d, err := declarationOf(engcomp.Composed{Node: "n1", Module: "dnsmasq", Resources: []json.RawMessage{svc, ctr}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := anyInactive(context.Background(), c.rt, liveness.LongRunning(d, nil), c.held); got != c.want {
|
||||||
|
t.Fatalf("inactive = %v, want %v", got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Joining through the tunnel (novox/hq ADR 0169).
|
||||||
|
//
|
||||||
|
// **The bus is never open to the internet, so a joining machine reaches it over the tunnel.** It
|
||||||
|
// makes its tunnel key first and prints the public half; the token is issued for that key, and the
|
||||||
|
// hub is told the key before the token is shown; the token carries the one peer this machine needs.
|
||||||
|
// So the tunnel can come up before the mesh has said anything else — the circle ADR 0004 broke by
|
||||||
|
// carrying the bus's address in the token is broken here by carrying the hub's.
|
||||||
|
|
||||||
|
// tunnelConfigPath and tunnelUnit are where the mesh's own declaration puts the private network, so
|
||||||
|
// the first tunnel is the same interface and unit the mesh takes over, not a second one beside it.
|
||||||
|
var (
|
||||||
|
tunnelConfigPath = "/etc/wireguard/mesh0.conf"
|
||||||
|
tunnelUnit = "wg-quick@mesh0"
|
||||||
|
// lookPath finds WireGuard's tools; a variable so a test needs none installed.
|
||||||
|
lookPath = exec.LookPath
|
||||||
|
)
|
||||||
|
|
||||||
|
// keyCommand makes this machine's tunnel key, or reads the one it already made, and prints the
|
||||||
|
// public half: what the token is issued for. Making it twice would be a token issued for a key the
|
||||||
|
// machine no longer has, so an existing key is kept.
|
||||||
|
func keyCommand(opts options) error {
|
||||||
|
path := identity.OverlayKeyPath(opts.state)
|
||||||
|
if key, err := identity.LoadOverlayKey(path); err == nil {
|
||||||
|
fmt.Println(key.Public)
|
||||||
|
return nil
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(identity.Path(opts.state)); err == nil {
|
||||||
|
return fmt.Errorf("this machine has joined already (%s), and its tunnel key is its own; "+
|
||||||
|
"there is no key to make", identity.Path(opts.state))
|
||||||
|
}
|
||||||
|
key, err := identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this machine's tunnel key: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Println(key.Public)
|
||||||
|
fmt.Fprintln(os.Stderr, "\nthis machine's tunnel key, made here; the private half stays in "+path+".\n"+
|
||||||
|
"Issue the token for it — `token issue --new <name> --overlay-key <the line above>` — and enrol with that token.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelKeyFor is the key a token through the tunnel was issued for, read from where `key` left it.
|
||||||
|
// Refused when there is none, or it is another: the hub knows only the key the token names.
|
||||||
|
func tunnelKeyFor(t *identity.TokenTunnel, state string) (identity.OverlayKey, error) {
|
||||||
|
path := identity.OverlayKeyPath(state)
|
||||||
|
key, err := identity.LoadOverlayKey(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return identity.OverlayKey{}, fmt.Errorf("this token was issued for a tunnel key, and this " +
|
||||||
|
"machine has none: run `nox-mesh-host key` here first and issue the token for the key it prints")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return identity.OverlayKey{}, err
|
||||||
|
}
|
||||||
|
if key.Public != t.Key {
|
||||||
|
return identity.OverlayKey{}, fmt.Errorf("this token was issued for the tunnel key %s, and this "+
|
||||||
|
"machine's is %s — it is another machine's token, or the key was made again; issue a new "+
|
||||||
|
"token for %s", t.Key, key.Public, key.Public)
|
||||||
|
}
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelConfig is the first tunnel: this machine's address, and the hub as its one peer, reaching the
|
||||||
|
// whole private network through it. The private key is set from its file, as the mesh's own
|
||||||
|
// declaration does it, so the file holds no secret.
|
||||||
|
func tunnelConfig(t *identity.TokenTunnel, keyPath string) string {
|
||||||
|
return fmt.Sprintf(`# Written by nox-mesh-host enrol: the one peer a joining machine needs (novox/hq ADR 0169).
|
||||||
|
# The mesh's own declaration replaces this once the machine has joined.
|
||||||
|
[Interface]
|
||||||
|
Address = %s
|
||||||
|
PostUp = wg set %%i private-key %s
|
||||||
|
|
||||||
|
[Peer]
|
||||||
|
PublicKey = %s
|
||||||
|
Endpoint = %s
|
||||||
|
AllowedIPs = %s
|
||||||
|
PersistentKeepalive = 25
|
||||||
|
`, t.Address, keyPath, t.HubKey, t.HubEndpoint, t.Range)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bringTheTunnelUp writes the first tunnel and starts it, so the bus the token names can be reached.
|
||||||
|
func bringTheTunnelUp(ctx context.Context, t *identity.TokenTunnel, keyPath string, run apply.Runner) error {
|
||||||
|
if _, err := lookPath("wg-quick"); err != nil {
|
||||||
|
return errors.New("joining through the tunnel needs WireGuard's tools on this machine " +
|
||||||
|
"(wireguard-tools), and wg-quick is not here")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(tunnelConfigPath), 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(tunnelConfigPath, []byte(tunnelConfig(t, keyPath)), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write the first tunnel: %w", err)
|
||||||
|
}
|
||||||
|
if out, err := run(ctx, "systemctl", "restart", tunnelUnit); err != nil {
|
||||||
|
return fmt.Errorf("the first tunnel would not start (%s): %v %s", tunnelUnit, err, strings.TrimSpace(out))
|
||||||
|
}
|
||||||
|
fmt.Printf("the tunnel to the hub is up: %s, through %s\n", t.Address, t.HubEndpoint)
|
||||||
|
return waitForTheHub(ctx, run, handshakeWithin)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handshakeWithin is how long the hub has to answer the first tunnel. Issuing the token sent the hub
|
||||||
|
// this machine as a peer; the hub applies that on its own time, and a bus dialled before it has is a
|
||||||
|
// timeout that names the bus rather than the tunnel.
|
||||||
|
var handshakeWithin = 90 * time.Second
|
||||||
|
|
||||||
|
// waitForTheHub waits until the tunnel has shaken hands with the hub, so the bus is dialled over a
|
||||||
|
// tunnel that answers — and says so in the tunnel's own words when it does not.
|
||||||
|
func waitForTheHub(ctx context.Context, run apply.Runner, within time.Duration) error {
|
||||||
|
deadline := time.Now().Add(within)
|
||||||
|
for {
|
||||||
|
out, err := run(ctx, "wg", "show", "mesh0", "latest-handshakes")
|
||||||
|
if err == nil {
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) == 2 && fields[1] != "0" {
|
||||||
|
fmt.Println("the hub answered the tunnel")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("the hub has not answered the tunnel in %s: the token may be another machine's, "+
|
||||||
|
"the hub may not have been sent this machine as a peer, or its tunnel's port is not reachable "+
|
||||||
|
"from here", within)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `key` makes the tunnel key once and prints its public half; asked again it prints the same one,
|
||||||
|
// because a token may already have been issued for it (novox/hq ADR 0169).
|
||||||
|
func TestKeyMakesTheTunnelKeyOnceAndKeepsIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
|
first := captureStdout(t, func() {
|
||||||
|
if err := keyCommand(opts); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
second := captureStdout(t, func() {
|
||||||
|
if err := keyCommand(opts); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if strings.TrimSpace(first) == "" || strings.TrimSpace(first) != strings.TrimSpace(second) {
|
||||||
|
t.Fatalf("the key changed between two asks: %q then %q", first, second)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(identity.OverlayKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("the private half is readable beyond root: %v", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A token through the tunnel takes the key it was issued for, and says so when this machine has none
|
||||||
|
// or another.
|
||||||
|
func TestATokenThroughTheTunnelTakesItsOwnKey(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
state := filepath.Join(dir, "state.json")
|
||||||
|
tt := &identity.TokenTunnel{Key: "x", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}
|
||||||
|
if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "nox-mesh-host key") {
|
||||||
|
t.Fatalf("a machine with no key was not told to make one: %v", err)
|
||||||
|
}
|
||||||
|
captureStdout(t, func() { _ = keyCommand(options{state: state}) })
|
||||||
|
if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "issued for the tunnel key x") {
|
||||||
|
t.Fatalf("another machine's token was taken: %v", err)
|
||||||
|
}
|
||||||
|
mine, _ := identity.LoadOverlayKey(identity.OverlayKeyPath(state))
|
||||||
|
tt.Key = mine.Public
|
||||||
|
if got, err := tunnelKeyFor(tt, state); err != nil || got.Public != mine.Public {
|
||||||
|
t.Fatalf("this machine's own token was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first tunnel is the mesh's interface and unit, with the hub as its one peer and no secret in
|
||||||
|
// the file — the same shape the mesh's declaration replaces it with.
|
||||||
|
func TestTheFirstTunnelIsTheMeshsInterfaceWithTheHubAsItsPeer(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tunnelConfigPath = filepath.Join(dir, "wireguard", "mesh0.conf")
|
||||||
|
lookPath = func(string) (string, error) { return "/usr/bin/wg-quick", nil }
|
||||||
|
t.Cleanup(func() { tunnelConfigPath = "/etc/wireguard/mesh0.conf" })
|
||||||
|
var ran []string
|
||||||
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
if name == "wg" {
|
||||||
|
return "HUBKEY\t1759400000\n", nil
|
||||||
|
}
|
||||||
|
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
tt := &identity.TokenTunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "HUBKEY", HubEndpoint: "198.51.100.1:51820"}
|
||||||
|
captureStdout(t, func() {
|
||||||
|
if err := bringTheTunnelUp(context.Background(), tt, "/var/lib/mesh-host/overlay.key", run); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
raw, err := os.ReadFile(tunnelConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := string(raw)
|
||||||
|
for _, want := range []string{"Address = 10.42.0.9/32", "PostUp = wg set %i private-key /var/lib/mesh-host/overlay.key",
|
||||||
|
"PublicKey = HUBKEY", "Endpoint = 198.51.100.1:51820", "AllowedIPs = 10.42.0.0/16", "PersistentKeepalive = 25"} {
|
||||||
|
if !strings.Contains(conf, want) {
|
||||||
|
t.Errorf("the first tunnel lacks %q:\n%s", want, conf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(conf, "PrivateKey") {
|
||||||
|
t.Error("the first tunnel's file holds the private key")
|
||||||
|
}
|
||||||
|
if len(ran) != 1 || ran[0] != "systemctl restart wg-quick@mesh0" {
|
||||||
|
t.Errorf("the tunnel was started as %v", ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// captureStdout is what fn printed to standard output.
|
||||||
|
func captureStdout(t *testing.T, fn func()) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
fn()
|
||||||
|
os.Stdout = was
|
||||||
|
w.Close()
|
||||||
|
out, _ := io.ReadAll(r)
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus is dialled only once the hub has answered the tunnel, and a hub that never does is said
|
||||||
|
// as the tunnel's fault rather than the bus's.
|
||||||
|
func TestTheBusWaitsForTheHubToAnswer(t *testing.T) {
|
||||||
|
asked := 0
|
||||||
|
answersOnThird := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
asked++
|
||||||
|
if asked < 3 {
|
||||||
|
return "HUBKEY\t0\n", nil
|
||||||
|
}
|
||||||
|
return "HUBKEY\t1759400000\n", nil
|
||||||
|
}
|
||||||
|
captureStdout(t, func() {
|
||||||
|
if err := waitForTheHub(context.Background(), answersOnThird, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
never := func(context.Context, string, ...string) (string, error) { return "HUBKEY\t0\n", nil }
|
||||||
|
err := waitForTheHub(context.Background(), never, 0)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has not answered the tunnel") {
|
||||||
|
t.Fatalf("a hub that never answered was not said: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which
|
||||||
|
// the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned.
|
||||||
|
func judgeDirectories(statePath string) (string, error) {
|
||||||
|
d, err := declaredNow(statePath, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
return "", errors.New("this node keeps no declaration, so it is declared no directory")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("what runs on this node does not read: %w", err)
|
||||||
|
}
|
||||||
|
return apply.JudgeDirectories(d), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeDirectoriesFrom is judgeDirectories as the controller asks it over the link.
|
||||||
|
func judgeDirectoriesFrom(statePath string) link.JudgeDirectoriesFunc {
|
||||||
|
return func(string) (string, error) { return judgeDirectories(statePath) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeDirectoriesCommand is `nox-mesh-host judge-directories` at this machine, as root: the same judgement, for a
|
||||||
|
// machine the controller cannot ask.
|
||||||
|
func judgeDirectoriesCommand(opts options) error {
|
||||||
|
said, err := judgeDirectories(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintln(opts.out, said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the
|
||||||
|
// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine
|
||||||
|
// reads them and checks them against the digest. Never another module's files: the seat's verb is the way.
|
||||||
|
|
||||||
|
// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object.
|
||||||
|
var (
|
||||||
|
keptAskWithin = 30 * time.Second
|
||||||
|
keptObjectWithin = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build
|
||||||
|
// holds the seat of the language it was built in, and the others do not answer.
|
||||||
|
var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat}
|
||||||
|
|
||||||
|
// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node.
|
||||||
|
func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error),
|
||||||
|
ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error),
|
||||||
|
object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch {
|
||||||
|
return func(ctx context.Context, module, digest string) ([]byte, bool, error) {
|
||||||
|
node, fp, kept, err := buildOf(ctx, module, digest)
|
||||||
|
if err != nil || !kept {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
hex := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
var notHeld []string
|
||||||
|
for _, seat := range keptBuildSeats {
|
||||||
|
raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact,
|
||||||
|
nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin)
|
||||||
|
if err != nil {
|
||||||
|
notHeld = append(notHeld, seat+": "+err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var got nodebuildgo.KeptArtifactAnswer
|
||||||
|
if json.Unmarshal(raw, &got) != nil {
|
||||||
|
return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case got.Error != "":
|
||||||
|
return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error)
|
||||||
|
case got.Refused != "":
|
||||||
|
return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words)
|
||||||
|
case got.Store == "" || got.Object == "" || got.Digest != hex:
|
||||||
|
return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex)
|
||||||
|
}
|
||||||
|
octx, cancel := context.WithTimeout(ctx, keptObjectWithin)
|
||||||
|
body, err := object(octx, got.Store, got.Object, got.Size+1)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err)
|
||||||
|
}
|
||||||
|
return body, true, nil
|
||||||
|
}
|
||||||
|
return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)",
|
||||||
|
node, module, strings.Join(notHeld, "; "))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47).
|
||||||
|
func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) {
|
||||||
|
hex := strings.Repeat("a", 64)
|
||||||
|
stored := func(string) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
kept bool
|
||||||
|
answer func(seat string) (json.RawMessage, error)
|
||||||
|
object error
|
||||||
|
wantKept bool
|
||||||
|
wantErr string
|
||||||
|
wantSeat string
|
||||||
|
}{
|
||||||
|
{"a build kept nothing is not asked", false, nil, nil, false, "", ""},
|
||||||
|
{"the seat of the language answers", true, func(seat string) (json.RawMessage, error) {
|
||||||
|
if seat != "node-build-node" {
|
||||||
|
return nil, errors.New("no responder")
|
||||||
|
}
|
||||||
|
return stored(seat)
|
||||||
|
}, nil, true, "", "node-build-node"},
|
||||||
|
{"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") },
|
||||||
|
nil, true, "novox keeps the build of distribution", ""},
|
||||||
|
{"the seat refuses it", true, func(string) (json.RawMessage, error) {
|
||||||
|
return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil
|
||||||
|
}, nil, true, "refuses the kept build of distribution: not kept", ""},
|
||||||
|
{"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
var seats []string
|
||||||
|
fetch := keptArtifactFetch(
|
||||||
|
func(context.Context, string, string) (string, string, bool, error) {
|
||||||
|
return "novox", "f1", row.kept, nil
|
||||||
|
},
|
||||||
|
func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) {
|
||||||
|
if node != "novox" || verb != "kept-artifact" {
|
||||||
|
t.Errorf("asked %s of %s on %s", verb, seat, node)
|
||||||
|
}
|
||||||
|
seats = append(seats, seat)
|
||||||
|
return row.answer(seat)
|
||||||
|
},
|
||||||
|
func(_ context.Context, store, name string, limit int64) ([]byte, error) {
|
||||||
|
if store != "kept-builds" || name != hex || limit < 5 {
|
||||||
|
t.Errorf("read %s/%s up to %d", store, name, limit)
|
||||||
|
}
|
||||||
|
return []byte("bytes"), row.object
|
||||||
|
})
|
||||||
|
body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex)
|
||||||
|
if kept != row.wantKept {
|
||||||
|
t.Fatalf("kept = %v", kept)
|
||||||
|
}
|
||||||
|
if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) {
|
||||||
|
t.Fatalf("got %q, %v", body, err)
|
||||||
|
}
|
||||||
|
if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) {
|
||||||
|
t.Fatalf("the error is %v, want %q", err, row.wantErr)
|
||||||
|
}
|
||||||
|
if !row.kept && len(seats) > 0 {
|
||||||
|
t.Errorf("a build that kept nothing asked %v", seats)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A library is judged on every look (novox/hq ADR 0328 §8, §9).
|
||||||
|
//
|
||||||
|
// Each library this machine is reached by is said as a resource `library:<name>` of kind library, with no module:
|
||||||
|
// healthy while it is kept (or, off its home, while its mount is there), waiting with an access wait of reason keep
|
||||||
|
// or disk, unknown while its filesystem could not be named, unhealthy otherwise. The root is the target, never in a
|
||||||
|
// reason nor in a wait's words: the controller says them on the operator's channel, which withholds a path (ADR
|
||||||
|
// 0234 §6). An access is judged by nothing, as before ADR 0325.
|
||||||
|
|
||||||
|
// libraryJudge reads every library on every look. Safe for the apply and the looking loop at once.
|
||||||
|
type libraryJudge struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
libraries []declaration.Library
|
||||||
|
declared *declaration.Declaration
|
||||||
|
// was is each library's last verdict, with since when and how many looks in a row found it unhealthy.
|
||||||
|
was map[string]link.ResourceHealth
|
||||||
|
|
||||||
|
// reading guards inFlight: every library whose look has not returned yet.
|
||||||
|
reading sync.Mutex
|
||||||
|
inFlight map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// accesses is the one library judge on this machine (its name from when it judged accesses).
|
||||||
|
var accesses = newLibraryJudge()
|
||||||
|
|
||||||
|
func newLibraryJudge() *libraryJudge {
|
||||||
|
return &libraryJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// set is what the declaration just applied declares: every library. One no longer declared is forgotten.
|
||||||
|
func (j *libraryJudge) set(d *declaration.Declaration, _ map[string]bool) {
|
||||||
|
var ls []declaration.Library
|
||||||
|
if d != nil {
|
||||||
|
ls = append(ls, d.Libraries...)
|
||||||
|
}
|
||||||
|
sort.Slice(ls, func(a, b int) bool { return ls[a].Library < ls[b].Library })
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
j.libraries, j.declared = ls, d
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, l := range ls {
|
||||||
|
declared[apply.LibraryAccess(l.Library)] = true
|
||||||
|
}
|
||||||
|
for id := range j.was {
|
||||||
|
if !declared[id] {
|
||||||
|
delete(j.was, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaration is the declaration last set.
|
||||||
|
func (j *libraryJudge) declaration() *declaration.Declaration {
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
return j.declared
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookBound is how long a look waits for one library to answer. A hard network mount whose server is down blocks
|
||||||
|
// a stat for ever, and one library must never stop every health statement this machine makes.
|
||||||
|
var lookBound = 5 * time.Second
|
||||||
|
|
||||||
|
// verdict is one library's judgement in a look.
|
||||||
|
type verdict struct {
|
||||||
|
state, reason string
|
||||||
|
waits []link.AccessWait
|
||||||
|
act *link.AccessActRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeBounded judges one library within lookBound, outside any lock. A look that has not answered is said so;
|
||||||
|
// while it still hangs, later looks start no other on the same library.
|
||||||
|
func (j *libraryJudge) judgeBounded(l declaration.Library) (verdict, bool) {
|
||||||
|
j.reading.Lock()
|
||||||
|
if j.inFlight[l.Library] {
|
||||||
|
j.reading.Unlock()
|
||||||
|
return verdict{}, false
|
||||||
|
}
|
||||||
|
j.inFlight[l.Library] = true
|
||||||
|
j.reading.Unlock()
|
||||||
|
done := make(chan verdict, 1)
|
||||||
|
go func() {
|
||||||
|
v := judgeLibrary(l)
|
||||||
|
j.reading.Lock()
|
||||||
|
delete(j.inFlight, l.Library)
|
||||||
|
j.reading.Unlock()
|
||||||
|
done <- v
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case v := <-done:
|
||||||
|
return v, true
|
||||||
|
case <-time.After(lookBound):
|
||||||
|
return verdict{}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeLibrary is one library's judgement now (ADR 0328 §7, §8, §9).
|
||||||
|
func judgeLibrary(l declaration.Library) verdict {
|
||||||
|
v := apply.JudgeLibrary(context.Background(), l)
|
||||||
|
access := apply.LibraryAccess(l.Library)
|
||||||
|
var out verdict
|
||||||
|
if a := v.Act; a != nil {
|
||||||
|
out.act = &link.AccessActRecord{Act: a.Act, Access: access, Warrant: a.Warrant, By: a.By, State: a.State,
|
||||||
|
Started: a.Started, Ended: a.Ended, Folders: a.Folders, Files: a.Files, Kept: a.Kept, Links: a.Links,
|
||||||
|
Other: a.Other, Mounts: a.Mounts, Shared: a.Shared, Linked: a.Linked, Stripped: a.Stripped, Why: a.Why}
|
||||||
|
}
|
||||||
|
switch v.Kind {
|
||||||
|
case apply.VerdictWaitKeep:
|
||||||
|
out.state, out.reason = link.StateWaiting, "its root holds data the mesh has not taken on the filesystem "+
|
||||||
|
"it is on now, and waits for the operator's word (novox/hq ADR 0328 §7, §8)"
|
||||||
|
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||||
|
What: apply.KeepWords(l.Library), Placed: v.Placed, Reason: link.WaitKeep}}
|
||||||
|
case apply.VerdictWaitDisk:
|
||||||
|
out.state, out.reason = link.StateWaiting, v.Why+" (novox/hq ADR 0328 §9)"
|
||||||
|
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||||
|
What: apply.DiskWords(l.Library), Reason: link.WaitDisk}}
|
||||||
|
case apply.VerdictUnknown:
|
||||||
|
// Not a fault and not the operator's: said as not known, and judged again on the next look.
|
||||||
|
out.state, out.reason = link.StateUnknown, v.Why+"; judged again on the next look"
|
||||||
|
case apply.VerdictBroken:
|
||||||
|
out.state, out.reason = link.StateUnhealthy, v.Why
|
||||||
|
default:
|
||||||
|
out.state = link.StateHealthy
|
||||||
|
switch {
|
||||||
|
case v.Act != nil && v.Act.State == link.ActRunning:
|
||||||
|
out.state, out.reason = link.StateStarting, "the take act is setting its group under its root"
|
||||||
|
case v.Act != nil && (v.Act.State == link.ActStopped || v.Act.State == link.ActFailed):
|
||||||
|
out.state, out.reason = link.StateStarting, "the take act goes on from where it stopped: "+v.Act.Why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// look reads every library now. Whether any verdict changed since the last look, for the loop to say it.
|
||||||
|
func (j *libraryJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
|
||||||
|
j.mu.Lock()
|
||||||
|
ls := append([]declaration.Library(nil), j.libraries...)
|
||||||
|
j.mu.Unlock()
|
||||||
|
|
||||||
|
read := make([]verdict, len(ls))
|
||||||
|
for i, l := range ls {
|
||||||
|
v, answered := j.judgeBounded(l)
|
||||||
|
if !answered {
|
||||||
|
v = verdict{state: link.StateUnknown, reason: "its root does not answer: judging it did not return " +
|
||||||
|
"within " + lookBound.String()}
|
||||||
|
}
|
||||||
|
read[i] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// And every tool runner's account the last apply could not put in a group, as a resource of its own (ADR 0334
|
||||||
|
// §2): unhealthy, never failing the apply.
|
||||||
|
warnings := apply.RunnerWarnings()
|
||||||
|
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
changed := false
|
||||||
|
out := make([]link.ResourceHealth, 0, len(ls)+len(warnings))
|
||||||
|
for _, w := range warnings {
|
||||||
|
v := link.ResourceHealth{Module: w.Module, Resource: w.Resource, Kind: link.KindRunner, State: link.StateUnhealthy,
|
||||||
|
Reason: "the tool runner's account joins no group: " + w.Why, Since: now.UTC(), Streak: 1}
|
||||||
|
if was, had := j.was[w.Resource]; had && was.Reason == v.Reason {
|
||||||
|
v.Since, v.Streak = was.Since, was.Streak+1
|
||||||
|
} else {
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
j.was[w.Resource] = v
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
for i, l := range ls {
|
||||||
|
r := read[i]
|
||||||
|
id := apply.LibraryAccess(l.Library)
|
||||||
|
v := link.ResourceHealth{Resource: id, Kind: link.KindLibrary, Target: l.Root, State: r.state,
|
||||||
|
Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act}
|
||||||
|
if was, had := j.was[id]; had && was.State == r.state && was.Reason == r.reason {
|
||||||
|
v.Since = was.Since
|
||||||
|
v.Streak = was.Streak
|
||||||
|
} else {
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
if r.state == link.StateUnhealthy {
|
||||||
|
v.Streak++
|
||||||
|
}
|
||||||
|
j.was[id] = v
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
return out, changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// withAccesses adds to a statement every library, as a resource of kind library.
|
||||||
|
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
|
||||||
|
h.Resources = append(h.Resources, vs...)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessesHealthy says no library in vs is anything but healthy.
|
||||||
|
func accessesHealthy(vs []link.ResourceHealth) bool {
|
||||||
|
for _, v := range vs {
|
||||||
|
if v.State != link.StateHealthy {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// keeper goes on with take acts stopped at their bound, beside the applies, one at a time (ADR 0328 §7).
|
||||||
|
type keeper struct {
|
||||||
|
running atomic.Bool
|
||||||
|
say func(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keeping is the one keeper on this machine.
|
||||||
|
var keeping = &keeper{say: func(string) {}}
|
||||||
|
|
||||||
|
// poke goes on with every stopped take act of the declaration last applied, unless that is running already.
|
||||||
|
func (k *keeper) poke(ctx context.Context) {
|
||||||
|
d := accesses.declaration()
|
||||||
|
if d == nil || len(d.Libraries) == 0 || !k.running.CompareAndSwap(false, true) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer k.running.Store(false)
|
||||||
|
for _, line := range apply.ReconcileLibraries(ctx, d) {
|
||||||
|
k.say(line)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// startKeeping has the keeper say what it does.
|
||||||
|
func startKeeping(ctx context.Context, say func(string)) {
|
||||||
|
keeping.say = say
|
||||||
|
keeping.poke(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessActFrom is the take act as the controller asks it over the link (ADR 0328 §7): judged against the
|
||||||
|
// declaration this node keeps, and run beside the applies.
|
||||||
|
func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
|
||||||
|
return func(act link.AccessAct) (string, error) {
|
||||||
|
d, err := declaredNow(statePath, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
return "", errors.New("this node keeps no declaration, so it is declared no library")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("what runs on this node does not read: %w", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
_, said, err := apply.TakeLibrary(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return said, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withWaiting is held with every id of waiting beside it: what liveness does not judge.
|
||||||
|
func withWaiting(held, waiting map[string]bool) map[string]bool {
|
||||||
|
if len(waiting) == 0 {
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
out := map[string]bool{}
|
||||||
|
for id := range held {
|
||||||
|
out[id] = true
|
||||||
|
}
|
||||||
|
for id := range waiting {
|
||||||
|
out[id] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||||
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A library whose root holds data never taken is said waiting with one access wait: the access `library:<name>`, the
|
||||||
|
// part its name, words with no path, the filesystem's name, and reason keep (novox/hq ADR 0328 §8); one whose root
|
||||||
|
// is absent waits with reason disk (§9). Neither carries a module.
|
||||||
|
func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||||
|
was := apply.Libraries
|
||||||
|
apply.Libraries = apply.NewLibraryMemory()
|
||||||
|
defer func() { apply.Libraries = was }()
|
||||||
|
dir := t.TempDir()
|
||||||
|
root := filepath.Join(dir, "media")
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "a film"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[{"name":"mesh-media","gid":62000},` +
|
||||||
|
`{"name":"mesh-books","gid":62001}],"libraries":[` +
|
||||||
|
`{"library":"media","root":"` + root + `","group":"mesh-media","gid":62000,"home":true},` +
|
||||||
|
`{"library":"books","root":"` + filepath.Join(dir, "books") + `","group":"mesh-books","gid":62001,"home":true}],` +
|
||||||
|
`"owns_nothing":true,"resources":[]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
j := newLibraryJudge()
|
||||||
|
j.set(d, nil)
|
||||||
|
vs, _ := j.look(time.Now())
|
||||||
|
if len(vs) != 2 {
|
||||||
|
t.Fatalf("the libraries are not said: %+v", vs)
|
||||||
|
}
|
||||||
|
books, media := vs[0], vs[1]
|
||||||
|
if media.Kind != link.KindLibrary || media.Resource != "library:media" || media.Module != "" ||
|
||||||
|
media.State != link.StateWaiting || len(media.AccessWaits) != 1 {
|
||||||
|
t.Fatalf("a library holding data is not said waiting: %+v", media)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(media.AccessWaits[0])
|
||||||
|
want := `{"kind":"access","access":"library:media","part":"media","what":"let the mesh keep the library media",` +
|
||||||
|
`"placed":"` + media.AccessWaits[0].Placed + `","reason":"keep"}`
|
||||||
|
if string(raw) != want || !strings.HasPrefix(media.AccessWaits[0].Placed, "fs-") {
|
||||||
|
t.Fatalf("the wait reads %s", raw)
|
||||||
|
}
|
||||||
|
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||||
|
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||||
|
}
|
||||||
|
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
|
||||||
|
// open, nor after it ended; and a library waiting for its disk is not asked.
|
||||||
|
t.Run("the home engine asks the take once", func(t *testing.T) {
|
||||||
|
var asked []oc.Ask
|
||||||
|
state := &link.MemState{}
|
||||||
|
h := &link.HandActs{Node: "n1", State: state,
|
||||||
|
JudgeAccess: func(act, access string) (link.Judged, error) {
|
||||||
|
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
|
||||||
|
},
|
||||||
|
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
|
||||||
|
h.Asker = &occ.Client{Module: occ.EngineAsker("n1"), State: state,
|
||||||
|
Publish: func(subject string, body any, id string) error {
|
||||||
|
if a, ok := body.(oc.Ask); ok {
|
||||||
|
asked = append(asked, a)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
waiting := keepWaiting(vs)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
|
||||||
|
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
|
||||||
|
}
|
||||||
|
h.Heard(oc.Warrant{Ask: asked[0].ID, Asker: occ.EngineAsker("n1"), Outcome: occ.OutcomeExpired, At: occ.FormatTime(time.Now())})
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
|
||||||
|
}
|
||||||
|
// Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again.
|
||||||
|
keys, _ := state.Keys("act.")
|
||||||
|
for _, k := range keys {
|
||||||
|
_ = state.Delete(k)
|
||||||
|
}
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again with the acts gone: %d", len(asked))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// loginsFile is where this engine keeps the logins it holds between runs (novox/hq ADR 0583 §1): beside its identity, readable by
|
||||||
|
// its owner alone, as the identity's membership password is.
|
||||||
|
func loginsFile(state string) string {
|
||||||
|
return filepath.Join(filepath.Dir(identity.Path(state)), "logins.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadLogins is the engine's logins: what it kept, with the membership's password as the first login's.
|
||||||
|
func loadLogins(state, node, first string) *link.Logins {
|
||||||
|
path := loginsFile(state)
|
||||||
|
var saved link.LoginsState
|
||||||
|
if raw, err := os.ReadFile(path); err == nil {
|
||||||
|
_ = json.Unmarshal(raw, &saved)
|
||||||
|
}
|
||||||
|
return link.NewLogins(node, first, saved, func(s link.LoginsState) error { return writeLogins(path, s) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLogins(path string, s link.LoginsState) error {
|
||||||
|
raw, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp := path + ".new"
|
||||||
|
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loginsFollowAgain bounds the wait before a watch of node-logins that could not be made is tried again on the same link:
|
||||||
|
// the vault may not serve rotation yet, and until it does this node is not granted the read.
|
||||||
|
var loginsFollowAgain = 2 * time.Minute
|
||||||
|
|
||||||
|
// followLogins follows this node's key of the vault's node-logins on every link, as a watch (never a poll), for as long as ctx
|
||||||
|
// lasts, and hands each value to hear. A watch that cannot be made is said once and tried again.
|
||||||
|
func followLogins(ctx context.Context, session *link.Session, node string, hear func(meshvault.NodeLoginsByNodeValue), say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
watching, stop := context.WithCancel(linked)
|
||||||
|
updates, werr := session.KVWatch(watching, buckets.NodeLogins, node)
|
||||||
|
if werr != nil {
|
||||||
|
stop()
|
||||||
|
if line := "this node's login is not followed yet (" + werr.Error() + "): it dials as it last did, and looks again in " + loginsFollowAgain.String(); line != said {
|
||||||
|
said = line
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
said = ""
|
||||||
|
for u := range updates {
|
||||||
|
if u.Caught || u.Deleted || len(u.Value) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nl, err := meshvaultclient.DecodeNodeLogin(u.Value)
|
||||||
|
if err != nil {
|
||||||
|
say("the key " + node + " of " + buckets.NodeLoginsRead + " is not read: " + err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hear(nl)
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
case <-time.After(loginsFollowAgain):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchLogin asks the vault, through its existing hand-out of a machine's own secrets, for the value of login, sealed to this
|
||||||
|
// node's key, and opens it with the sealing key; no new verb is asked for it.
|
||||||
|
func fetchLogin(queue *link.Session, statePath string) func(context.Context, string) (string, error) {
|
||||||
|
unseal := sealOpener(statePath)
|
||||||
|
return func(ctx context.Context, login string) (string, error) {
|
||||||
|
// The vault names the login values under the engine's own module, which is §527's (§634).
|
||||||
|
own, err := queue.Own.Wait(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the node-engine's own module is not named yet, so the vault's name of its login value is not known: %w", err)
|
||||||
|
}
|
||||||
|
raw, err := queue.AskVault(ctx, "own", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Own []struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
|
} `json:"own"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return "", fmt.Errorf("the vault's own answer is not one: %w", err)
|
||||||
|
}
|
||||||
|
for _, o := range a.Own {
|
||||||
|
if o.Module != own || o.Name != login {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sealed := string(o.Sealed)
|
||||||
|
var text string
|
||||||
|
if json.Unmarshal(o.Sealed, &text) == nil {
|
||||||
|
sealed = text
|
||||||
|
}
|
||||||
|
value, err := unseal(sealed)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the value of %s cannot be opened with this node's sealing key: %w", login, err)
|
||||||
|
}
|
||||||
|
return string(value), nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), own, login)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func queueNode(q *link.Session) string { return q.Membership.Node }
|
||||||
|
|
||||||
|
// newRotator is the engine's side of a rotation over this session (ADR 0583 §3-§6): the value fetched through the vault's own
|
||||||
|
// hand-out, the test-dial on a second connection, the confirmation over the new link, and a fall back carried in the health
|
||||||
|
// report.
|
||||||
|
func newRotator(queue *link.Session, statePath string, timeout time.Duration, say link.Announce) *link.Rotator {
|
||||||
|
r := &link.Rotator{Logins: queue.Membership.Logins, Say: say,
|
||||||
|
Fetch: fetchLogin(queue, statePath),
|
||||||
|
TestDial: func(ctx context.Context, login, password string) error {
|
||||||
|
return link.TestDial(ctx, queue.Membership, login, password, timeout)
|
||||||
|
},
|
||||||
|
Confirm: func(ctx context.Context) error {
|
||||||
|
login, _ := queue.Membership.Logins.Current()
|
||||||
|
if err := queue.WhenLinkedAs(ctx, login); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := queue.AskVault(ctx, "node-account-confirm", map[string]any{})
|
||||||
|
return err
|
||||||
|
}}
|
||||||
|
r.Watch()
|
||||||
|
// The fall back is carried in this node's health report, from which the watchman raises node-account-fallback (ADR 0583 §6).
|
||||||
|
queue.Fallback = r.Fallback
|
||||||
|
return r
|
||||||
|
}
|
||||||
+974
-685
File diff suppressed because it is too large
Load Diff
+57
-296
@@ -3,19 +3,17 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/bundle"
|
|
||||||
"github.com/novox/mesh-host/internal/declaration"
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
@@ -98,26 +96,6 @@ func TestNoCommandIsNotAnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyNeedsExactlyOneDeclaration(t *testing.T) {
|
|
||||||
// `apply` takes a file where every other command takes nothing, so the leftover-argument
|
|
||||||
// rule has an exception — and an exception is where a parser stops refusing things it
|
|
||||||
// should. Both directions are checked.
|
|
||||||
if _, _, err := parseArgs([]string{"apply"}); err == nil {
|
|
||||||
t.Error("apply with no file was accepted")
|
|
||||||
}
|
|
||||||
if _, _, err := parseArgs([]string{"apply", "a.json", "b.json"}); err == nil {
|
|
||||||
t.Error("apply with two files was accepted")
|
|
||||||
}
|
|
||||||
|
|
||||||
command, opts, err := parseArgs([]string{"apply", "decl.json", "--dry-run"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if command != "apply" || opts.file != "decl.json" || !opts.dryRun {
|
|
||||||
t.Errorf("parsed as command=%q file=%q dry-run=%v", command, opts.file, opts.dryRun)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheStateHasADocumentedDefault(t *testing.T) {
|
func TestTheStateHasADocumentedDefault(t *testing.T) {
|
||||||
// A host that wrote its state somewhere unexpected would forget what it owns on the next
|
// A host that wrote its state somewhere unexpected would forget what it owns on the next
|
||||||
// run, and then leave everything it had applied behind forever.
|
// run, and then leave everything it had applied behind forever.
|
||||||
@@ -135,97 +113,77 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
|
|||||||
// off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the
|
// off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the
|
||||||
// standard library stops at the first non-flag argument wherever that argument is.
|
// standard library stops at the first non-flag argument wherever that argument is.
|
||||||
for _, args := range [][]string{
|
for _, args := range [][]string{
|
||||||
{"apply", "decl.json", "--dry-run", "--json"},
|
{"hand-over", "decl.json", "--dry-run", "--json"},
|
||||||
{"apply", "--dry-run", "decl.json", "--json"},
|
{"hand-over", "--dry-run", "decl.json", "--json"},
|
||||||
{"apply", "--dry-run", "--json", "decl.json"},
|
{"hand-over", "--dry-run", "--json", "decl.json"},
|
||||||
} {
|
} {
|
||||||
command, opts, err := parseArgs(args)
|
command, opts, err := parseArgs(args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("%v: unexpected error: %v", args, err)
|
t.Errorf("%v: unexpected error: %v", args, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if command != "apply" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
|
if command != "hand-over" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
|
||||||
t.Errorf("%v parsed as file=%q dry-run=%v json=%v",
|
t.Errorf("%v parsed as file=%q dry-run=%v json=%v",
|
||||||
args, opts.file, opts.dryRun, opts.json)
|
args, opts.file, opts.dryRun, opts.json)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds
|
|
||||||
// or its firewall changed — which is how a predecessor still writing is caught, without a report
|
|
||||||
// every five minutes saying nothing new.
|
|
||||||
func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
|
|
||||||
w := &adoptionWatch{}
|
|
||||||
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}}
|
|
||||||
if !w.changed(held) {
|
|
||||||
t.Fatal("the first report of a hold was not said")
|
|
||||||
}
|
|
||||||
again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}}
|
|
||||||
if w.changed(again) {
|
|
||||||
t.Error("the same holds in another order were said again")
|
|
||||||
}
|
|
||||||
rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}}
|
|
||||||
if !w.changed(rewritten) {
|
|
||||||
t.Error("a held file rewritten by something else was not said")
|
|
||||||
}
|
|
||||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
|
|
||||||
t.Error("a changed firewall was not said")
|
|
||||||
}
|
|
||||||
// What filters the machine is part of it (novox/hq ADR 0168): a predecessor's chain removed by
|
|
||||||
// hand, or the found firewall enabled again, is said without being asked.
|
|
||||||
filtered := link.Report{Firewall: "none", Held: rewritten.Held,
|
|
||||||
Filters: []link.Filter{{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"}}}
|
|
||||||
if !w.changed(filtered) {
|
|
||||||
t.Error("a filter appearing was not said")
|
|
||||||
}
|
|
||||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
|
|
||||||
t.Error("a filter removed was not said")
|
|
||||||
}
|
|
||||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held, FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: true}}) {
|
|
||||||
t.Error("the found firewall coming back was not said")
|
|
||||||
}
|
|
||||||
if !worthSaying(link.Report{Filters: filtered.Filters}) {
|
|
||||||
t.Error("a report carrying only what filters the machine is not worth saying")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue
|
|
||||||
// tells the watch every report the broker took (link.Queue.Heard).
|
|
||||||
func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) {
|
|
||||||
w := &adoptionWatch{}
|
|
||||||
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
|
|
||||||
heard := w.said
|
|
||||||
heard(report)
|
|
||||||
if w.changed(report) {
|
|
||||||
t.Error("a reconcile repeated what the mesh had just heard")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) {
|
|
||||||
// The controller previews a flip from what the node last said is reachable; a port that opened
|
|
||||||
// since must reach it without waiting for the next delivery (novox/hq ADR 0100).
|
|
||||||
w := &adoptionWatch{}
|
|
||||||
before := link.Report{Firewall: "ufw", Reachable: []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
|
|
||||||
if !w.changed(before) {
|
|
||||||
t.Fatal("the first report was not said")
|
|
||||||
}
|
|
||||||
reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{
|
|
||||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
|
|
||||||
if w.changed(reordered) {
|
|
||||||
t.Error("the same reachable set was said again")
|
|
||||||
}
|
|
||||||
opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable,
|
|
||||||
link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})}
|
|
||||||
if !w.changed(opened) {
|
|
||||||
t.Error("a newly published port was not said")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
|
// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
|
||||||
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
||||||
// what the other recorded.
|
// what the other recorded.
|
||||||
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
||||||
|
// Issue 1585: the engine's stop waits for the apply in flight to end and save, and then lets no save start; before,
|
||||||
|
// the process ended under it and a save cut before its rename left its temporary file and the state before it.
|
||||||
|
t.Run("the stop waits for the apply in flight and its save", func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
applying.Lock()
|
||||||
|
var said []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
waiting := make(chan struct{})
|
||||||
|
say := func(line string) {
|
||||||
|
mu.Lock()
|
||||||
|
said = append(said, line)
|
||||||
|
mu.Unlock()
|
||||||
|
if strings.Contains(line, "waiting up to") {
|
||||||
|
close(waiting)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stopped := make(chan func(), 1)
|
||||||
|
go func() { stopped <- stopBetweenApplies(say, time.Hour) }()
|
||||||
|
// Said before it blocks, and it cannot end before the apply lets go: so it has not ended now.
|
||||||
|
<-waiting
|
||||||
|
select {
|
||||||
|
case release := <-stopped:
|
||||||
|
release()
|
||||||
|
applying.Unlock()
|
||||||
|
t.Fatal("the stop did not wait for the apply in flight")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
applying.Unlock()
|
||||||
|
release := <-stopped
|
||||||
|
saved := make(chan error, 1)
|
||||||
|
go func() { saved <- store.Save(path, store.State{}) }()
|
||||||
|
// A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look.
|
||||||
|
select {
|
||||||
|
case <-saved:
|
||||||
|
release()
|
||||||
|
t.Fatal("a save started after the stop")
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
if err := <-saved; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") {
|
||||||
|
t.Errorf("said %q", said)
|
||||||
|
}
|
||||||
|
})
|
||||||
// A host is built for one system at link time, and a test binary has no link time: this asks
|
// A host is built for one system at link time, and a test binary has no link time: this asks
|
||||||
// the machine it runs on, and stands aside where the answer is no.
|
// the machine it runs on, and stands aside where the answer is no.
|
||||||
built, err := system.For("arch")
|
built, err := system.For("arch")
|
||||||
@@ -270,24 +228,6 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued
|
|
||||||
// and lost — the link down when the reconcile spoke — it must be said again.
|
|
||||||
func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
|
|
||||||
w := &adoptionWatch{}
|
|
||||||
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}}
|
|
||||||
if !w.differs(held) {
|
|
||||||
t.Fatal("the first report of a change was not new")
|
|
||||||
}
|
|
||||||
// The link was down: nothing published it, so nothing says it was said.
|
|
||||||
if !w.differs(held) {
|
|
||||||
t.Error("a change that never reached the mesh was counted as said")
|
|
||||||
}
|
|
||||||
w.said(held)
|
|
||||||
if w.differs(held) {
|
|
||||||
t.Error("a change the mesh was told was said again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
|
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
|
||||||
// the point of application, whichever command delivered it, naming both — an adopted control-node
|
// the point of application, whichever command delivered it, naming both — an adopted control-node
|
||||||
// once applied its converged genesis bundle and closed itself for forty-five minutes.
|
// once applied its converged genesis bundle and closed itself for forty-five minutes.
|
||||||
@@ -468,125 +408,6 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
|
|
||||||
// that — never to the bundle the host carries, which genesis consumed.
|
|
||||||
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
|
|
||||||
was := builtFor
|
|
||||||
builtFor = "arch"
|
|
||||||
t.Cleanup(func() { builtFor = was })
|
|
||||||
opts := stateWithMode(t, store.ModeAdopted)
|
|
||||||
|
|
||||||
controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
|
||||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
|
|
||||||
Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
|
|
||||||
_, _, _, err = reconcileSource(opts)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
|
|
||||||
t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
mine, err := identity.Generate("workstation")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
|
|
||||||
Signer: controllerPublic, Password: "issued"}
|
|
||||||
if err := identity.Save(identity.Path(opts.state), mine); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
d, raw, from, err := reconcileSource(opts)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
|
|
||||||
t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
|
|
||||||
}
|
|
||||||
|
|
||||||
// And before the mesh has said anything: the bundle, as it always was. A test binary carries
|
|
||||||
// only the placeholder, and asking for it is what proves the path.
|
|
||||||
if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _, _, err = reconcileSource(opts)
|
|
||||||
if !errors.Is(err, bundle.ErrEmpty) {
|
|
||||||
t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A machine says which links face outside without being asked.**
|
|
||||||
//
|
|
||||||
// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
|
|
||||||
// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
|
|
||||||
// could speak only when a declaration arrived — and a declaration cannot be composed until it has
|
|
||||||
// spoken. A machine waiting for a push that is waiting for the machine.
|
|
||||||
func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
|
|
||||||
w := &adoptionWatch{}
|
|
||||||
first := link.Report{Firewall: "none"}
|
|
||||||
if !w.differs(first) {
|
|
||||||
t.Fatal("the first report should differ from nothing")
|
|
||||||
}
|
|
||||||
w.said(first)
|
|
||||||
|
|
||||||
// Only the links changed, and nothing about adoption.
|
|
||||||
learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
|
|
||||||
if !w.differs(learnt) {
|
|
||||||
t.Fatal("a machine that has just learnt which links face outside would never say so, " +
|
|
||||||
"and could then never be sent a filter")
|
|
||||||
}
|
|
||||||
w.said(learnt)
|
|
||||||
if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
|
|
||||||
t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
|
|
||||||
// filter is written around the old one until it is.
|
|
||||||
if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
|
|
||||||
t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A converged machine can say which links face outside, unasked.**
|
|
||||||
//
|
|
||||||
// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
|
|
||||||
// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
|
|
||||||
// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
|
|
||||||
// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
|
|
||||||
// sat silent while the control plane refused to send them a filter.
|
|
||||||
func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
|
|
||||||
// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
|
|
||||||
converged := link.Report{Outward: []string{"eth0"}}
|
|
||||||
if !worthSaying(converged) {
|
|
||||||
t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
|
|
||||||
"sent a filter — a machine waiting for a push that is waiting for the machine")
|
|
||||||
}
|
|
||||||
|
|
||||||
// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
|
|
||||||
if !worthSaying(link.Report{Firewall: "ufw"}) {
|
|
||||||
t.Fatal("an adopted machine no longer says which firewall it found")
|
|
||||||
}
|
|
||||||
if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
|
|
||||||
t.Fatal("an adopted machine no longer says what it holds")
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
|
|
||||||
// about nothing.
|
|
||||||
if worthSaying(link.Report{}) {
|
|
||||||
t.Fatal("a reconcile with nothing to say speaks anyway")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A refused report says nothing about the machine; the refusal is for the console.
|
|
||||||
if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
|
|
||||||
t.Fatal("a refused report is offered as news about the machine")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A host running as a service says what its apply did.**
|
// **A host running as a service says what its apply did.**
|
||||||
//
|
//
|
||||||
// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
|
// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
|
||||||
@@ -611,63 +432,3 @@ func TestTheApplysLogIsNeverNil(t *testing.T) {
|
|||||||
t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
|
t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defends novox/hq issues 257 and 261: a reconcile due while the link applies a newer declaration
|
|
||||||
// applies that newer one once its turn comes — never the one kept when its timer fired. Read before
|
|
||||||
// waiting, it gave back a module assigned a second earlier, and reported a declaration the mesh no
|
|
||||||
// longer recorded as sent.
|
|
||||||
func TestAReconcileAppliesWhatWasKeptWhenItsTurnComes(t *testing.T) {
|
|
||||||
built, err := system.For("arch")
|
|
||||||
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
|
|
||||||
t.Skip("this machine is not one these tests can apply on")
|
|
||||||
}
|
|
||||||
was := builtFor
|
|
||||||
builtFor = "arch"
|
|
||||||
t.Cleanup(func() { builtFor = was })
|
|
||||||
dir := t.TempDir()
|
|
||||||
opts := options{state: filepath.Join(dir, "state.json")}
|
|
||||||
public, private, err := ed25519.GenerateKey(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
target := filepath.Join(dir, "a.conf")
|
|
||||||
kept := func(content string) store.Declared {
|
|
||||||
body := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + target +
|
|
||||||
`","content":"` + content + `\n"}]}`)
|
|
||||||
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
|
|
||||||
}
|
|
||||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("older")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The link is applying: the reconcile's timer fires and it waits its turn.
|
|
||||||
applying.Lock()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := reapplyKept(context.Background(), opts, public, nil, nil)
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
time.Sleep(50 * time.Millisecond)
|
|
||||||
// The link's apply ends by keeping the newer declaration, and lets go.
|
|
||||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("newer")); err != nil {
|
|
||||||
applying.Unlock()
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
applying.Unlock()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("the reconcile never ran once the node was free")
|
|
||||||
}
|
|
||||||
got, err := os.ReadFile(target)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(got) != "newer\n" {
|
|
||||||
t.Errorf("the reconcile applied %q, what was kept before the newer declaration", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// markWithin bounds one marking of the haves (what the composer tells its agent: a refusal, the words it keeps). The marks
|
||||||
|
// read and write each module's have over the bus one after the other; a bus that answers slowly or not at all must hold
|
||||||
|
// the marking, never the composition pass (issue 1965, shanks 2026-11-05 22:22: a pass waited minutes on them).
|
||||||
|
var markWithin = 45 * time.Second
|
||||||
|
|
||||||
|
// marker runs the composer's marks of the haves off the composition pass: set returns at once, one worker runs what was
|
||||||
|
// set last under each name, in the order the names were first set, each within markWithin. A mark set while one runs
|
||||||
|
// replaces the pending mark of its name, so the haves are marked once for the newest word, not once for every pass.
|
||||||
|
type marker struct {
|
||||||
|
run context.Context
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]func(context.Context)
|
||||||
|
order []string
|
||||||
|
working bool
|
||||||
|
idle *sync.Cond
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMarker(run context.Context) *marker {
|
||||||
|
m := &marker{run: run, pending: map[string]func(context.Context){}}
|
||||||
|
m.idle = sync.NewCond(&m.mu)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// set takes the mark name should carry; it never waits on the bus.
|
||||||
|
func (m *marker) set(name string, mark func(context.Context)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if _, queued := m.pending[name]; !queued {
|
||||||
|
m.order = append(m.order, name)
|
||||||
|
}
|
||||||
|
m.pending[name] = mark
|
||||||
|
if m.working {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.working = true
|
||||||
|
go m.work()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *marker) work() {
|
||||||
|
for {
|
||||||
|
m.mu.Lock()
|
||||||
|
if len(m.order) == 0 || m.run.Err() != nil {
|
||||||
|
m.order, m.pending = nil, map[string]func(context.Context){}
|
||||||
|
m.working = false
|
||||||
|
m.idle.Broadcast()
|
||||||
|
m.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name := m.order[0]
|
||||||
|
m.order = m.order[1:]
|
||||||
|
mark := m.pending[name]
|
||||||
|
delete(m.pending, name)
|
||||||
|
m.mu.Unlock()
|
||||||
|
ctx, cancel := context.WithTimeout(m.run, markWithin)
|
||||||
|
mark(ctx)
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait returns once no mark is pending or running (for tests).
|
||||||
|
func (m *marker) wait() {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
for m.working {
|
||||||
|
m.idle.Wait()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1965 (shanks 2026-11-05 22:22): the marks of the haves never hold the composition pass. A mark that stalls on the
|
||||||
|
// bus is cut at markWithin, a mark set while one runs replaces the pending one of its name, and set itself never waits.
|
||||||
|
func marksOfTheHavesNeverHoldThePass(t *testing.T) {
|
||||||
|
// The bound is the test's own: the stalled mark is released by a channel, never by a real wait.
|
||||||
|
setMarkWithin(t, time.Hour)
|
||||||
|
m := newMarker(context.Background())
|
||||||
|
release := make(chan struct{})
|
||||||
|
var ran []string
|
||||||
|
stalled := make(chan struct{})
|
||||||
|
m.set("refused", func(ctx context.Context) { // a have write that stalls until the test lets it go
|
||||||
|
close(stalled)
|
||||||
|
<-release
|
||||||
|
ran = append(ran, "refused-stalled")
|
||||||
|
})
|
||||||
|
<-stalled
|
||||||
|
// Set while the first runs: only the newest of a name is kept.
|
||||||
|
m.set("kept", func(context.Context) { ran = append(ran, "kept-old") })
|
||||||
|
m.set("kept", func(context.Context) { ran = append(ran, "kept-new") })
|
||||||
|
m.set("refused", func(context.Context) { ran = append(ran, "refused-new") })
|
||||||
|
// All four sets above returned while the first mark still stalled: set never waits on it.
|
||||||
|
close(release)
|
||||||
|
m.wait()
|
||||||
|
want := []string{"refused-stalled", "kept-new", "refused-new"}
|
||||||
|
if len(ran) != len(want) {
|
||||||
|
t.Fatalf("ran %v; want %v", ran, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if ran[i] != want[i] {
|
||||||
|
t.Fatalf("ran %v; want %v", ran, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMarkWithin(t *testing.T, d time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
old := markWithin
|
||||||
|
markWithin = d
|
||||||
|
t.Cleanup(func() { markWithin = old })
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1132/AC1, AC3: the report carries one fact per module the declaration names, with the build the controller sent for
|
||||||
|
// it; a module not yet taken on an adopted node is not one.
|
||||||
|
func TestTheReportCarriesOneFactPerModuleWithItsBuild(t *testing.T) {
|
||||||
|
d, err := declaration.Parse([]byte(`{"declaration":1,"builds":{"a":"b-1","b":"b-2"},"resources":[
|
||||||
|
{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
|
||||||
|
{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
failure := &apply.Error{Resource: "b.dir", Err: errors.New("no space"), Failures: []apply.Failure{{Resource: "b.dir", Err: errors.New("no space")}}}
|
||||||
|
facts := moduleFacts(d, apply.Report{}, failure)
|
||||||
|
if len(facts) != 2 || facts[0].Module != "a" || facts[0].Build != "b-1" || facts[0].Failed ||
|
||||||
|
facts[1].Module != "b" || facts[1].Build != "b-2" || !facts[1].Failed || facts[1].Detail == "" {
|
||||||
|
t.Fatalf("facts %+v", facts)
|
||||||
|
}
|
||||||
|
adopted, err := declaration.Parse([]byte(`{"declaration":1,"adoption":{"taken":["a"],"untaken":{"b":["b.dir"]}},"resources":[
|
||||||
|
{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
|
||||||
|
{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if facts := moduleFacts(adopted, apply.Report{}, nil); len(facts) != 1 || facts[0].Module != "a" {
|
||||||
|
t.Fatalf("an untaken module is a fact: %+v", facts)
|
||||||
|
}
|
||||||
|
if moduleFacts(nil, apply.Report{}, nil) != nil {
|
||||||
|
t.Fatal("facts without a declaration")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
osuser "os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestAMoveOutDirectoryKeepsOnlyWhatItKnows drives the node-engine whole through its command (issue 1024, novox/hq
|
||||||
|
// ADR 0422): a directory under an account's home that says move-out keeps the file the mesh declared and moves a
|
||||||
|
// file nobody declared into the kept folder, named in a list, and the run says it moved it. It needs no bus: the
|
||||||
|
// apply is the one a delivery and the five-minute reconcile both run.
|
||||||
|
func TestAMoveOutDirectoryKeepsOnlyWhatItKnows(t *testing.T) {
|
||||||
|
me, err := osuser.Current()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The user database lists an account as a person's only at uid 1000 or more; root, as the forge's merge check
|
||||||
|
// runs, is listed under a number of its own (it can give files to it), by the same name.
|
||||||
|
listedUID := me.Uid
|
||||||
|
if uid, _ := strconv.Atoi(me.Uid); uid < 1000 || uid == 65534 {
|
||||||
|
listedUID = "4242"
|
||||||
|
}
|
||||||
|
root := t.TempDir()
|
||||||
|
home := filepath.Join(root, "home", "agent")
|
||||||
|
dir := filepath.Join(home, ".ssh")
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
passwd := filepath.Join(root, "passwd")
|
||||||
|
line := fmt.Sprintf("%s:x:%s:%s::%s:/bin/sh\n", me.Username, listedUID, me.Gid, home)
|
||||||
|
if err := os.WriteFile(passwd, []byte(line), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(apply.UseAccountsFileForTest(passwd))
|
||||||
|
// Independent of the machine: an arch host with a pacman on PATH that answers and holds nothing, as
|
||||||
|
// TestReplay233 puts one there (it answers with an error); nothing here declares a package.
|
||||||
|
bin := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
wasBuilt := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = wasBuilt })
|
||||||
|
|
||||||
|
stray := filepath.Join(dir, "id_stray")
|
||||||
|
if err := os.WriteFile(stray, []byte("somebody else's key"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := time.Now().Add(-time.Hour)
|
||||||
|
if err := os.Chtimes(stray, old, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pipe := filepath.Join(dir, "agent.sock")
|
||||||
|
if err := syscall.Mkfifo(pipe, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chtimes(pipe, old, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts := options{state: filepath.Join(root, "state.json")}
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"ssh.dir","type":"directory","path":"` + dir + `","owner":"` + me.Username + `","unknown":"move-out"},` +
|
||||||
|
`{"id":"ssh.config","type":"file","path":"` + dir + `/config","content":"Host *\n","owner":"` + me.Username + `"}]}`)
|
||||||
|
|
||||||
|
var said []string
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(s string) { said = append(said, s) })
|
||||||
|
|
||||||
|
if report.Refused != "" || len(report.Failed) > 0 {
|
||||||
|
t.Fatalf("refused or failed: %q %v", report.Refused, report.Failed)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "config")); err != nil {
|
||||||
|
t.Errorf("the declared file is not there: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(stray); err == nil {
|
||||||
|
t.Error("the file nobody declared stayed in the directory")
|
||||||
|
}
|
||||||
|
lists, _ := filepath.Glob(filepath.Join(home, ".local", "state", "mesh-moved", "ssh", "*.json"))
|
||||||
|
if len(lists) != 1 {
|
||||||
|
t.Fatalf("one list expected: %v", lists)
|
||||||
|
}
|
||||||
|
rawList, err := os.ReadFile(lists[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var entries []struct{ From, To string }
|
||||||
|
if err := json.Unmarshal(rawList, &entries); err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("the list: %s %v", rawList, err)
|
||||||
|
}
|
||||||
|
if got, err := os.ReadFile(entries[0].To); err != nil || string(got) != "somebody else's key" {
|
||||||
|
t.Errorf("the moved file is not whole in the kept folder: %q %v", got, err)
|
||||||
|
}
|
||||||
|
if len(report.MovedOut) != 1 || report.MovedOut[0].From != stray || report.MovedOut[0].To != entries[0].To {
|
||||||
|
t.Errorf("the report does not carry the move: %+v", report.MovedOut)
|
||||||
|
}
|
||||||
|
if len(report.Left) != 1 || report.Left[0].Path != pipe {
|
||||||
|
t.Errorf("the report does not list the pipe left in place: %+v", report.Left)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(said, "\n"), "moved "+stray) {
|
||||||
|
t.Errorf("the run does not say it moved the file: %v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/subtle"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/nacl/box"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The move of a value sealed to this machine into the vault (design 57's V3, issue 676; internal/link
|
||||||
|
// movesecret.go). The controller cannot open what it holds; this machine can open what was sealed to it, and so
|
||||||
|
// hands it to the vault, sealed to a key the vault made for the call. The value is in this process for that one
|
||||||
|
// call and written nowhere.
|
||||||
|
|
||||||
|
// vaultAsk asks one of the vault's verbs in this engine's own name and answers its result.
|
||||||
|
type vaultAsk func(verb string, args any) (json.RawMessage, error)
|
||||||
|
|
||||||
|
// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over
|
||||||
|
// the link open now.
|
||||||
|
func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc {
|
||||||
|
return func(m link.MoveSecret) (link.MovedSecret, error) {
|
||||||
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, err
|
||||||
|
}
|
||||||
|
ask := func(verb string, args any) (json.RawMessage, error) {
|
||||||
|
return queue.AskVault(context.Background(), verb, args)
|
||||||
|
}
|
||||||
|
moved, err := moveIntoTheVault(ask, key, m)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
||||||
|
}
|
||||||
|
if moved.Superseded != "" {
|
||||||
|
if _, said := supersededSaid.LoadOrStore(moved.Secret, true); !said {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: %s\n", moved.Superseded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return moved, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// moveIntoTheVault opens the row's value with this machine's sealing key, hands it to the vault's move-in in its two
|
||||||
|
// steps, and answers what the vault kept once its read-back — sealed to a key made here for this one call — opens
|
||||||
|
// to the same value.
|
||||||
|
func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret) (link.MovedSecret, error) {
|
||||||
|
if m.SealedTo != "" && m.SealedTo != key.Public {
|
||||||
|
return link.MovedSecret{}, errors.New("the controller's store says this value was sealed to a sealing key this " +
|
||||||
|
"machine no longer has, so it cannot be opened here")
|
||||||
|
}
|
||||||
|
value, err := key.Unseal(m.Sealed)
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the value does not open with this machine's sealing key: %w", err)
|
||||||
|
}
|
||||||
|
defer wipe(value)
|
||||||
|
|
||||||
|
// This machine's sealing key, stated in its own name: the key the vault seals the core's credentials to for the
|
||||||
|
// controller to carry here (novox/hq ADR 0349 §16, issue 763), never a key the controller names.
|
||||||
|
first := map[string]any{"origin": m.Origin, "generation": m.Generation, "sealing_key": key.Public}
|
||||||
|
if !m.MadeAt.IsZero() {
|
||||||
|
first["made_at"] = m.MadeAt.UTC().Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
if m.Own() {
|
||||||
|
first["node"], first["module"], first["name"] = m.Node, m.Module, m.Name
|
||||||
|
} else {
|
||||||
|
first["provider"], first["provision"] = m.Provider, m.Provision
|
||||||
|
first["consumer"], first["consumer_module"] = m.Consumer, m.ConsumerModule
|
||||||
|
if m.Local != "" {
|
||||||
|
first["local"] = m.Local
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw, err := ask("move-in", first)
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault did not open the move: %w", err)
|
||||||
|
}
|
||||||
|
var opened struct {
|
||||||
|
Call string `json:"call"`
|
||||||
|
SealTo string `json:"seal_to"`
|
||||||
|
Secret string `json:"secret"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
|
||||||
|
return link.MovedSecret{}, errors.New("the vault answered no call and no key to seal to")
|
||||||
|
}
|
||||||
|
sealed, err := identity.Seal(opened.SealTo, value)
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
|
||||||
|
}
|
||||||
|
backPub, backPriv, err := box.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, err
|
||||||
|
}
|
||||||
|
defer wipe(backPriv[:])
|
||||||
|
raw, err = ask("move-in", map[string]any{"call": opened.Call, "sealed": sealed,
|
||||||
|
"read_back_to": base64.StdEncoding.EncodeToString(backPub[:])})
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault did not take %s: %w", opened.Secret, err)
|
||||||
|
}
|
||||||
|
var kept struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
Unchanged bool `json:"unchanged"`
|
||||||
|
ReadBack string `json:"read_back"`
|
||||||
|
// Refused is the vault's refusal as fields (mesh-vault's MoveRefused, mesh-catalog f3b13de2): nothing was taken.
|
||||||
|
Refused *struct {
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
Held *heldValue `json:"held"`
|
||||||
|
} `json:"refused"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's answer for %s does not read: %v", opened.Secret, err)
|
||||||
|
}
|
||||||
|
if r := kept.Refused; r != nil {
|
||||||
|
// The vault holds another value for the name, and the move never replaces one: the vault's value is the live
|
||||||
|
// one, and this machine's older value is superseded, never asked again (the sweep of 2026-10-30: it failed
|
||||||
|
// "answered with no read-back" on every ask).
|
||||||
|
if r.Reason == vaultHoldsAnother && r.Held != nil && r.Held.Name != "" {
|
||||||
|
return link.MovedSecret{Secret: r.Held.Name, Generation: r.Held.Generation, Superseded: r.Held.words()}, nil
|
||||||
|
}
|
||||||
|
why := kept.Why
|
||||||
|
if why == "" {
|
||||||
|
why = r.Reason
|
||||||
|
}
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault refused %s: %s", opened.Secret, why)
|
||||||
|
}
|
||||||
|
if kept.Name == "" || kept.ReadBack == "" {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
||||||
|
}
|
||||||
|
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
||||||
|
if err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s is not sealed", kept.Name)
|
||||||
|
}
|
||||||
|
back, ok := box.OpenAnonymous(nil, blob, backPub, backPriv)
|
||||||
|
if !ok {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s does not open with the key made for it", kept.Name)
|
||||||
|
}
|
||||||
|
defer wipe(back)
|
||||||
|
if subtle.ConstantTimeCompare(back, value) != 1 {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault keeps %s, but it read back another value than the one handed "+
|
||||||
|
"in: the move of it did not hold", kept.Name)
|
||||||
|
}
|
||||||
|
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// vaultHoldsAnother is the vault's reason when it holds another value for the name (mesh-vault RefusedHeldAnother).
|
||||||
|
const vaultHoldsAnother = "holds-another-value"
|
||||||
|
|
||||||
|
// heldValue is the record of the value the vault holds instead, never the value (mesh-vault HeldAnother).
|
||||||
|
type heldValue struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
ChangedBy string `json:"changed_by"`
|
||||||
|
ChangedAt time.Time `json:"changed_at"`
|
||||||
|
Origin string `json:"origin"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// words say the held value's record, for the journal and the controller.
|
||||||
|
func (h heldValue) words() string {
|
||||||
|
w := fmt.Sprintf("the vault holds another value for %s, generation %d, changed by %s at %s", h.Name, h.Generation,
|
||||||
|
h.ChangedBy, h.ChangedAt.UTC().Format(time.RFC3339))
|
||||||
|
if h.Origin != "" {
|
||||||
|
w += ", origin " + h.Origin
|
||||||
|
}
|
||||||
|
return w + ": that value is the live one, and this machine's older value is superseded and not moved"
|
||||||
|
}
|
||||||
|
|
||||||
|
// supersededSaid holds the names a superseded move was said of, so the journal says each once.
|
||||||
|
var supersededSaid sync.Map
|
||||||
|
|
||||||
|
func wipe(b []byte) {
|
||||||
|
for i := range b {
|
||||||
|
b[i] = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/nacl/box"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
|
||||||
|
// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
|
||||||
|
type standInVault struct {
|
||||||
|
t *testing.T
|
||||||
|
pub *[32]byte
|
||||||
|
priv *[32]byte
|
||||||
|
opened map[string]any
|
||||||
|
kept string
|
||||||
|
lie string
|
||||||
|
refuse string
|
||||||
|
// answer replaces the second step's answer: the vault's refusal as fields (mesh-catalog f3b13de2).
|
||||||
|
answer map[string]any
|
||||||
|
carried []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
|
||||||
|
raw, _ := json.Marshal(args)
|
||||||
|
v.carried = append(v.carried, string(raw))
|
||||||
|
if verb != "move-in" {
|
||||||
|
v.t.Fatalf("asked %s", verb)
|
||||||
|
}
|
||||||
|
if v.refuse != "" {
|
||||||
|
return nil, errors.New(v.refuse)
|
||||||
|
}
|
||||||
|
var a map[string]any
|
||||||
|
_ = json.Unmarshal(raw, &a)
|
||||||
|
if a["call"] == nil {
|
||||||
|
v.opened = a
|
||||||
|
pub, priv, _ := box.GenerateKey(rand.Reader)
|
||||||
|
v.pub, v.priv = pub, priv
|
||||||
|
return json.Marshal(map[string]string{"call": "c1", "seal_to": base64.StdEncoding.EncodeToString(pub[:]),
|
||||||
|
"secret": "own/anchor/postgres/superuser-password"})
|
||||||
|
}
|
||||||
|
blob, _ := base64.StdEncoding.DecodeString(a["sealed"].(string))
|
||||||
|
value, ok := box.OpenAnonymous(nil, blob, v.pub, v.priv)
|
||||||
|
if !ok {
|
||||||
|
v.t.Fatal("what the engine sealed does not open with the call's key")
|
||||||
|
}
|
||||||
|
v.kept = string(value)
|
||||||
|
if v.answer != nil {
|
||||||
|
return json.Marshal(v.answer)
|
||||||
|
}
|
||||||
|
back := v.kept
|
||||||
|
if v.lie != "" {
|
||||||
|
back = v.lie
|
||||||
|
}
|
||||||
|
key, _ := base64.StdEncoding.DecodeString(a["read_back_to"].(string))
|
||||||
|
var to [32]byte
|
||||||
|
copy(to[:], key)
|
||||||
|
boxed, _ := box.SealAnonymous(nil, []byte(back), &to, rand.Reader)
|
||||||
|
return json.Marshal(map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 1,
|
||||||
|
"read_back": base64.StdEncoding.EncodeToString(boxed)})
|
||||||
|
}
|
||||||
|
|
||||||
|
const theValue = "the-superusers-password"
|
||||||
|
|
||||||
|
func aMoveOf(t *testing.T, key identity.SealingKey) link.MoveSecret {
|
||||||
|
t.Helper()
|
||||||
|
sealed, err := identity.Seal(key.Public, []byte(theValue))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return link.MoveSecret{Node: "anchor", By: "the move", MoveSecretAsk: link.MoveSecretAsk{Module: "postgres",
|
||||||
|
Name: "superuser-password", Origin: "made", Generation: "g-1", Sealed: sealed, SealedTo: key.Public}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The engine opens what was sealed to its machine, hands it to the vault sealed to the call's key — never in the
|
||||||
|
// clear — and answers what the vault kept only once the read-back opens to the same value.
|
||||||
|
func TestTheEngineMovesAValueSealedToItAndChecksTheReadBack(t *testing.T) {
|
||||||
|
key, err := identity.GenerateSealingKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
vault := &standInVault{t: t}
|
||||||
|
moved, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if moved.Secret != "own/anchor/postgres/superuser-password" || moved.Generation != 1 || vault.kept != theValue {
|
||||||
|
t.Fatalf("moved %+v, the vault kept %q", moved, vault.kept)
|
||||||
|
}
|
||||||
|
if vault.opened["module"] != "postgres" || vault.opened["name"] != "superuser-password" || vault.opened["node"] != "anchor" ||
|
||||||
|
vault.opened["origin"] != "made" || vault.opened["generation"] != "g-1" ||
|
||||||
|
vault.opened["sealing_key"] != key.Public {
|
||||||
|
t.Errorf("the first step named %v", vault.opened)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.Join(vault.carried, " "), theValue) {
|
||||||
|
t.Fatal("the value crossed the bus in the clear")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A read-back that is not the value handed in is a failed move, said; so is a value this machine cannot open, and
|
||||||
|
// the vault's refusal.
|
||||||
|
func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
|
||||||
|
key, _ := identity.GenerateSealingKey()
|
||||||
|
vault := &standInVault{t: t, lie: "something-else"}
|
||||||
|
if _, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "read back") {
|
||||||
|
t.Fatalf("a read-back of another value: %v", err)
|
||||||
|
}
|
||||||
|
other, _ := identity.GenerateSealingKey()
|
||||||
|
if _, err := moveIntoTheVault((&standInVault{t: t}).ask, key, aMoveOf(t, other)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "sealing key") {
|
||||||
|
t.Fatalf("a value sealed to another key: %v", err)
|
||||||
|
}
|
||||||
|
refusing := &standInVault{t: t, refuse: "the move never replaces a value the vault holds"}
|
||||||
|
if _, err := moveIntoTheVault(refusing.ask, key, aMoveOf(t, key)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "never replaces") {
|
||||||
|
t.Fatalf("the vault's refusal: %v", err)
|
||||||
|
}
|
||||||
|
// The vault's refusal as fields (mesh-catalog f3b13de2, no read-back): for another value held, the vault's is the
|
||||||
|
// live one and the move is done, superseded, at the held generation; any other reason fails in the vault's words.
|
||||||
|
held := map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 3,
|
||||||
|
"changed_by": "mesh-vault@novox", "changed_at": "2026-10-30T15:50:00Z", "origin": "made"}
|
||||||
|
for _, row := range []struct {
|
||||||
|
refused map[string]any
|
||||||
|
gen int
|
||||||
|
fails string
|
||||||
|
}{
|
||||||
|
{map[string]any{"reason": "holds-another-value", "held": held}, 3, ""},
|
||||||
|
{map[string]any{"reason": "locked"}, 0, "the vault refused"},
|
||||||
|
} {
|
||||||
|
v := &standInVault{t: t, answer: map[string]any{"refused": row.refused, "why": "in words"}}
|
||||||
|
moved, err := moveIntoTheVault(v.ask, key, aMoveOf(t, key))
|
||||||
|
switch {
|
||||||
|
case row.fails != "" && (err == nil || !strings.Contains(err.Error(), row.fails)):
|
||||||
|
t.Errorf("refused %v: %v", row.refused["reason"], err)
|
||||||
|
case row.fails == "" && (err != nil || moved.Generation != row.gen || moved.Secret != held["name"] ||
|
||||||
|
!strings.Contains(moved.Superseded, "changed by mesh-vault@novox")):
|
||||||
|
t.Errorf("refused %v: moved %+v, %v", row.refused["reason"], moved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A pair credential names both its ends.
|
||||||
|
pair := aMoveOf(t, key)
|
||||||
|
pair.Module, pair.Name = "", ""
|
||||||
|
pair.Provider, pair.Provision, pair.Consumer, pair.ConsumerModule = "anchor", "database", "shanks", "gitea"
|
||||||
|
ends := &standInVault{t: t}
|
||||||
|
if _, err := moveIntoTheVault(ends.ask, key, pair); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ends.opened["provider"] != "anchor" || ends.opened["consumer"] != "shanks" || ends.opened["consumer_module"] != "gitea" ||
|
||||||
|
ends.opened["node"] != nil {
|
||||||
|
t.Errorf("the pair's first step named %v", ends.opened)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/network"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The machine's networking in the engine's statement (novox/hq ADR 0241): the file judged is the one the
|
||||||
|
// uplink holder declares whole, the mesh name asked is the bus's, and the statement carries the parts.
|
||||||
|
|
||||||
|
func TestTheResolverFileJudgedIsTheOneAModuleDeclaresWhole(t *testing.T) {
|
||||||
|
d := &declaration.Declaration{Resources: []declaration.Resource{
|
||||||
|
&declaration.File{ID: "hosts.file", Type: "file", Path: "/etc/hosts", Content: "x"},
|
||||||
|
&declaration.File{ID: "networkmanager.resolv", Type: "file", Path: network.ResolvConf, Content: "nameserver 10.10.0.1\n"},
|
||||||
|
}}
|
||||||
|
content, owner, ok := declaredResolvConf(d)
|
||||||
|
if !ok || owner != "networkmanager" || content != "nameserver 10.10.0.1\n" {
|
||||||
|
t.Fatalf("got %q %q %v", content, owner, ok)
|
||||||
|
}
|
||||||
|
d.Resources[1].(*declaration.File).CreateOnce = true
|
||||||
|
if _, _, ok := declaredResolvConf(d); ok {
|
||||||
|
t.Fatal("a seed nobody holds the machine to was judged as the declared file")
|
||||||
|
}
|
||||||
|
if _, _, ok := declaredResolvConf(nil); ok {
|
||||||
|
t.Fatal("no declaration declared a file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheMeshNameAskedIsTheBuses(t *testing.T) {
|
||||||
|
for broker, want := range map[string]string{"anchor.internal:4222": "anchor.internal", "192.0.2.10:5671": "",
|
||||||
|
"anchor.internal": "anchor.internal", "[2001:db8::1]:4222": ""} {
|
||||||
|
if got := hostOf(broker); got != want {
|
||||||
|
t.Errorf("%s: got %q, want %q", broker, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheStatementCarriesTheNetwork(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||||
|
h := healthAsReported(liveness.Statement{At: at}, &network.Statement{State: network.Unhealthy, Since: at,
|
||||||
|
Parts: []network.Part{{Part: network.PartResolvConf, State: network.Unhealthy, Writer: "FortiClient",
|
||||||
|
Owner: "networkmanager", Reason: "the resolver file was rewritten by another program", Since: at}}})
|
||||||
|
if h.Network == nil || h.Network.State != network.Unhealthy || len(h.Network.Parts) != 1 ||
|
||||||
|
h.Network.Parts[0].Writer != "FortiClient" || h.Network.Parts[0].Owner != "networkmanager" {
|
||||||
|
t.Fatalf("the statement says %+v", h.Network)
|
||||||
|
}
|
||||||
|
if h := healthAsReported(liveness.Statement{At: at}, nil); h.Network != nil {
|
||||||
|
t.Fatal("an engine with no network judge said a network")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The checker runs gofmt, go vet and both test levels by Go's convention, and the build module starts the bus the
|
||||||
|
// integration level needs (novox/hq ADR 0529): no script of the repository's own runs beside them.
|
||||||
|
func TestTheRepositoryHoldsNoCheckScript(t *testing.T) {
|
||||||
|
if _, err := os.Stat(filepath.Join("..", "..", "merge-check.sh")); err == nil {
|
||||||
|
t.Fatal("merge-check.sh is back: the checker's levels run the tests")
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,47 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R6-A AC8: nothing in the node-engine receives, orders or reconciles a whole declaration (novox/hq ADR 0539 §9, ADR
|
||||||
|
// 0429 §8): no package subscribes to a node's declare subject or names the deleted symbols.
|
||||||
|
func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) {
|
||||||
|
deleted := []string{
|
||||||
|
"DeclareSubject", "mesh.node.\" + node + \".declare", "ReconcileDue", "ReconcileEvery", "olderThanKept",
|
||||||
|
"refuseOlder", "olderGeneration", "appliedGeneration", "reconcileKept", "reapplyKept", "adoptionWatch",
|
||||||
|
"unsaidFile", "numbersFile", "func gather(", "func handleBody(", "Supersedes(", "ReportSequence",
|
||||||
|
"RefusedOlder", "ReadsGeneration", "ReadsModules", "GenerationRefusal", "LeftOutModuleOf",
|
||||||
|
}
|
||||||
|
root := filepath.Join("..", "..")
|
||||||
|
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The composition moved in from mesh-sdk (novox/hq ADR 0593 §7) has a node's input field of the same name,
|
||||||
|
// ReadsModules, which says the node reads a module per resource; it is no reconciling of a declaration.
|
||||||
|
if d.IsDir() && path == filepath.Join(root, "internal", "compose") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, word := range deleted {
|
||||||
|
if strings.Contains(string(body), word) {
|
||||||
|
t.Errorf("%s still names %q", path, word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This machine's sealing key, stated to the vault in the engine's own name (novox/hq ADR 0372, issue 786). The vault
|
||||||
|
// records it from the machine's own account alone and, when it is a first or a changed key, says the machine's core
|
||||||
|
// secrets changed so the controller asks for them again sealed to it. Without this a machine that never moved a
|
||||||
|
// secret in (adopted after feature 57) has no key at the vault. Nothing but the public half crosses.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// nodeKeyRetryFrom and nodeKeyRetryUpTo bound the wait between tries while the vault does not answer: it starts
|
||||||
|
// after the bus and may not serve the verb yet.
|
||||||
|
nodeKeyRetryFrom = 5 * time.Second
|
||||||
|
nodeKeyRetryUpTo = 5 * time.Minute
|
||||||
|
// nodeKeyLook is how often the key file is read to see whether it changed.
|
||||||
|
nodeKeyLook = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
||||||
|
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is.
|
||||||
|
type heardKey struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
pem string
|
||||||
|
}
|
||||||
|
|
||||||
|
// vaultKey is the vault's key this engine heard.
|
||||||
|
var vaultKey = &heardKey{}
|
||||||
|
|
||||||
|
func (h *heardKey) get() string {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.pem
|
||||||
|
}
|
||||||
|
|
||||||
|
// heard keeps the key a node-key answer gave. An answer with no key (a vault before this one) changes nothing.
|
||||||
|
func (h *heardKey) heard(answer json.RawMessage) {
|
||||||
|
var a struct {
|
||||||
|
VaultKey string `json:"vault_key"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.pem = a.VaultKey
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
||||||
|
// vault takes the same key again and changes nothing (ADR 0547 clause 9).
|
||||||
|
const nodeKeyReask = 10 * time.Minute
|
||||||
|
|
||||||
|
// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes
|
||||||
|
// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault
|
||||||
|
// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed
|
||||||
|
// vault key is heard.
|
||||||
|
type keyHearing struct {
|
||||||
|
heard func(json.RawMessage)
|
||||||
|
held func() bool
|
||||||
|
// look is how often it is stated again while no vault key is held; every how often once one is.
|
||||||
|
look, every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
|
||||||
|
// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
|
||||||
|
// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With
|
||||||
|
// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval.
|
||||||
|
func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
|
||||||
|
retryFrom, retryUpTo time.Duration, hearing *keyHearing) {
|
||||||
|
stated, lastFailure := "", ""
|
||||||
|
var askedAt time.Time
|
||||||
|
wait := retryFrom
|
||||||
|
fail := func(why string) {
|
||||||
|
if why != lastFailure {
|
||||||
|
say("mesh-host: the sealing key was not stated to the vault, trying again: " + why)
|
||||||
|
lastFailure = why
|
||||||
|
}
|
||||||
|
wait = min(wait*2, retryUpTo)
|
||||||
|
}
|
||||||
|
look := min(nodeKeyLook, retryFrom)
|
||||||
|
for {
|
||||||
|
delay := look
|
||||||
|
key, err := load()
|
||||||
|
again := false
|
||||||
|
if hearing != nil && stated != "" {
|
||||||
|
since := time.Since(askedAt)
|
||||||
|
held := hearing.held == nil || hearing.held()
|
||||||
|
again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
fail(err.Error())
|
||||||
|
delay = wait
|
||||||
|
case key.Public != stated || again:
|
||||||
|
if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
|
||||||
|
fail(err.Error())
|
||||||
|
delay = wait
|
||||||
|
} else {
|
||||||
|
askedAt = time.Now()
|
||||||
|
if hearing != nil && hearing.heard != nil {
|
||||||
|
hearing.heard(answer)
|
||||||
|
}
|
||||||
|
if key.Public != stated {
|
||||||
|
say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
|
||||||
|
}
|
||||||
|
stated, lastFailure, wait = key.Public, "", retryFrom
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 786 (novox/hq ADR 0372): the engine states its own sealing key to the vault's node-key at start, retries
|
||||||
|
// while the vault does not answer, and states it again when the key changes; the same key is not stated twice.
|
||||||
|
type keyVault struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
stated []string
|
||||||
|
fail int
|
||||||
|
verbs []string
|
||||||
|
// keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the
|
||||||
|
// vault's key the others carry.
|
||||||
|
keyless int
|
||||||
|
key string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
|
||||||
|
v.mu.Lock()
|
||||||
|
defer v.mu.Unlock()
|
||||||
|
v.verbs = append(v.verbs, verb)
|
||||||
|
if v.fail > 0 {
|
||||||
|
v.fail--
|
||||||
|
return nil, errors.New("the vault does not answer")
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(args)
|
||||||
|
var a struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(raw, &a)
|
||||||
|
v.stated = append(v.stated, a.Key)
|
||||||
|
if v.key == "" || v.keyless > 0 {
|
||||||
|
v.keyless--
|
||||||
|
return json.RawMessage(`{"recorded":true}`), nil
|
||||||
|
}
|
||||||
|
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key})
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *keyVault) said() []string {
|
||||||
|
v.mu.Lock()
|
||||||
|
defer v.mu.Unlock()
|
||||||
|
return append([]string(nil), v.stated...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitFor(t *testing.T, what string, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
for i := 0; i < 400; i++ {
|
||||||
|
if ok() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("timed out: %s", what)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(t *testing.T) {
|
||||||
|
first, _ := identity.GenerateSealingKey()
|
||||||
|
second, _ := identity.GenerateSealingKey()
|
||||||
|
var mu sync.Mutex
|
||||||
|
current := first
|
||||||
|
load := func() (identity.SealingKey, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return current, nil
|
||||||
|
}
|
||||||
|
vault := &keyVault{fail: 2}
|
||||||
|
var lines []string
|
||||||
|
var lmu sync.Mutex
|
||||||
|
say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() }
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var hmu sync.Mutex
|
||||||
|
var answers []string
|
||||||
|
heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() }
|
||||||
|
go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard})
|
||||||
|
|
||||||
|
waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
|
||||||
|
// Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing.
|
||||||
|
waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 })
|
||||||
|
if vault.said()[0] != first.Public {
|
||||||
|
t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public)
|
||||||
|
}
|
||||||
|
vault.mu.Lock()
|
||||||
|
verbs := append([]string(nil), vault.verbs...)
|
||||||
|
vault.mu.Unlock()
|
||||||
|
for _, v := range verbs {
|
||||||
|
if v != "node-key" {
|
||||||
|
t.Errorf("asked %s", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The same key again is not stated again.
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
if n := len(vault.said()); n != 1 {
|
||||||
|
t.Errorf("the same key was stated %d times", n)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
current = second
|
||||||
|
mu.Unlock()
|
||||||
|
waitFor(t, "the changed key stated", func() bool { return len(vault.said()) == 2 })
|
||||||
|
if vault.said()[1] != second.Public {
|
||||||
|
t.Errorf("stated %q after the change, want %q", vault.said()[1], second.Public)
|
||||||
|
}
|
||||||
|
lmu.Lock()
|
||||||
|
defer lmu.Unlock()
|
||||||
|
if len(lines) == 0 {
|
||||||
|
t.Error("the engine said nothing")
|
||||||
|
}
|
||||||
|
asksAgainUntilItHearsTheVaultsKey(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key
|
||||||
|
// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is
|
||||||
|
// held, again only at the slow interval, so a changed vault key is heard; never more.
|
||||||
|
func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
|
||||||
|
sealing, _ := identity.GenerateSealingKey()
|
||||||
|
load := func() (identity.SealingKey, error) { return sealing, nil }
|
||||||
|
pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" }
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
keyless int
|
||||||
|
every time.Duration
|
||||||
|
change bool
|
||||||
|
wantAtLeast int
|
||||||
|
wantAtMost int
|
||||||
|
}{
|
||||||
|
{name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4},
|
||||||
|
{name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond,
|
||||||
|
change: true, wantAtLeast: 3, wantAtMost: 1 << 20},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
vault := &keyVault{keyless: c.keyless, key: pem("AAAA")}
|
||||||
|
h := &heardKey{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond,
|
||||||
|
&keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every})
|
||||||
|
waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") })
|
||||||
|
if c.change {
|
||||||
|
vault.mu.Lock()
|
||||||
|
vault.key = pem("BBBB")
|
||||||
|
vault.mu.Unlock()
|
||||||
|
waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") })
|
||||||
|
}
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
n := len(vault.said())
|
||||||
|
if n < c.wantAtLeast || n > c.wantAtMost {
|
||||||
|
t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
||||||
|
// answer gives is kept; an answer without one (an older vault) or with no public key changes nothing (ADR 0547 clause
|
||||||
|
// 9).
|
||||||
|
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
||||||
|
if link.NodeKeyContract != 13 {
|
||||||
|
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
||||||
|
}
|
||||||
|
h := &heardKey{}
|
||||||
|
h.heard(json.RawMessage(`{"recorded":true}`))
|
||||||
|
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
||||||
|
if h.get() != "" {
|
||||||
|
t.Fatalf("an answer with no key was kept: %q", h.get())
|
||||||
|
}
|
||||||
|
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
||||||
|
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
||||||
|
h.heard(answer)
|
||||||
|
if h.get() != pem {
|
||||||
|
t.Fatalf("key %q, want the key", h.get())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
|
|
||||||
// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
|
|
||||||
// "older" could not.
|
|
||||||
|
|
||||||
func keptWith(t *testing.T, sequence int64) store.Declared {
|
|
||||||
t.Helper()
|
|
||||||
return keptAt(t, link.Order{Sequence: sequence})
|
|
||||||
}
|
|
||||||
|
|
||||||
func keptAt(t *testing.T, order link.Order) store.Declared {
|
|
||||||
t.Helper()
|
|
||||||
body, err := json.Marshal(map[string]any{
|
|
||||||
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence,
|
|
||||||
"epoch": order.Epoch,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return store.Declared{Declaration: body, Signature: []byte("x")}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
|
|
||||||
err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("sequence 5 was accepted over a kept 7")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
|
|
||||||
t.Fatalf("the refusal does not say what it is: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
|
|
||||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil {
|
|
||||||
t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
|
|
||||||
}
|
|
||||||
// Equal is the same declaration again, which reconciling is for.
|
|
||||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil {
|
|
||||||
t.Fatalf("the same sequence was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
|
|
||||||
// An older controller sends none; a host that received before it understood them kept none.
|
|
||||||
// Refusing on a guess would strand a node the moment the controller is older than the host.
|
|
||||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil {
|
|
||||||
t.Fatalf("a declaration claiming no order was refused: %v", err)
|
|
||||||
}
|
|
||||||
if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil {
|
|
||||||
t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
|
|
||||||
}
|
|
||||||
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil {
|
|
||||||
t.Fatalf("a first declaration was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost
|
|
||||||
// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence.
|
|
||||||
func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) {
|
|
||||||
kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3})
|
|
||||||
err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") {
|
|
||||||
t.Fatalf("an older epoch was not refused by name: %v", err)
|
|
||||||
}
|
|
||||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil {
|
|
||||||
t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err)
|
|
||||||
}
|
|
||||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil {
|
|
||||||
t.Fatal("a lower sequence under the same epoch was accepted")
|
|
||||||
}
|
|
||||||
// A controller with no lease — older, or rolled back to a build from before it — is today's
|
|
||||||
// behaviour: no epoch compared.
|
|
||||||
if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil {
|
|
||||||
t.Fatalf("a declaration with no epoch was refused for one: %v", err)
|
|
||||||
}
|
|
||||||
if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older ||
|
|
||||||
held != (link.Order{Epoch: 57, Sequence: 3}) {
|
|
||||||
t.Fatalf("olderThanKept said %v, holding %+v", older, held)
|
|
||||||
}
|
|
||||||
// What is kept cannot be read: no order to compare, so nothing is refused on a guess.
|
|
||||||
if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil,
|
|
||||||
link.Order{Epoch: 41, Sequence: 1}); older {
|
|
||||||
t.Fatal("refused against a kept declaration nobody can read")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Design 47 G §527, §634, A-149: the node-engine's own module is the module of its applied record at its own fingerprint,
|
||||||
|
// and its finding writer is made, observing as that module, once the name is known: at once from the record, else when the
|
||||||
|
// first composing names it. No name is fixed in code.
|
||||||
|
func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testing.T) {
|
||||||
|
fp := "src2:" + strings.Repeat("a", 64)
|
||||||
|
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
|
||||||
|
for module, f := range map[string]string{"node": "", "the-engine": fp, "other": "src2:" + strings.Repeat("b", 64)} {
|
||||||
|
if err := applied.Keep(module, []byte(`{"module":"`+module+`","fp":"`+f+`"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, tc := range []struct{ fp, want string }{{fp, "the-engine"}, {"", ""}, {"src2:" + strings.Repeat("c", 64), ""}} {
|
||||||
|
if got := ownModuleApplied(applied, tc.fp); got != tc.want {
|
||||||
|
t.Errorf("the own module at %q is %q, want %q", tc.fp, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
own := link.NewOwnName()
|
||||||
|
made := make(chan string, 1)
|
||||||
|
late := &lateFindings{own: own, make: func(module string) (*findingsOfTheNode, error) {
|
||||||
|
made <- module
|
||||||
|
return nil, context.Canceled // the writer itself is the SDK's: only when it is made is tested here
|
||||||
|
}}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
var said []string
|
||||||
|
late.Start(ctx, func(s string) { said = append(said, s) })
|
||||||
|
if err := late.Write(ctx, condseat.Finding{}); err == nil || late.Held() != nil {
|
||||||
|
t.Fatalf("a finding was taken before the engine's own module was known: %v", err)
|
||||||
|
}
|
||||||
|
if len(made) != 0 {
|
||||||
|
t.Fatal("the writer was made before a name")
|
||||||
|
}
|
||||||
|
own.Set("the-engine") // the first composing names it
|
||||||
|
select {
|
||||||
|
case m := <-made:
|
||||||
|
if m != "the-engine" || late.observer() != "the-engine" {
|
||||||
|
t.Fatalf("the writer was made as %q, observing as %q", m, late.observer())
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the writer was never made after the name was known")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// §633: the engine's bound for the first seat list is the file the manifest's `view-waits-minutes` feeds, five minutes
|
||||||
|
// when it holds nothing usable.
|
||||||
|
func TestTheViewWaitsBoundIsTheManifestsSetting(t *testing.T) {
|
||||||
|
was := viewWaitsFile
|
||||||
|
defer func() { viewWaitsFile = was }()
|
||||||
|
for content, want := range map[string]time.Duration{"2\n": 2 * time.Minute, "": 5 * time.Minute, "0": 5 * time.Minute, "x": 5 * time.Minute} {
|
||||||
|
viewWaitsFile = filepath.Join(t.TempDir(), "w")
|
||||||
|
if err := os.WriteFile(viewWaitsFile, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := viewWaitsBound(); got != want {
|
||||||
|
t.Errorf("the file %q gives %v, want %v", content, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
viewWaitsFile = filepath.Join(t.TempDir(), "missing")
|
||||||
|
if got := viewWaitsBound(); got != 5*time.Minute {
|
||||||
|
t.Errorf("no file gives %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os/exec"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
// packageManagers are the tools a system's probe asks, for a system whose probe is its package manager.
|
||||||
|
var packageManagers = map[string]string{"arch": "pacman", "alpine": "apk"}
|
||||||
|
|
||||||
|
// unanswered is a probe of the package manager that failed before an apply: which tool, and its words.
|
||||||
|
type unanswered struct {
|
||||||
|
tool string
|
||||||
|
words string
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedKey is how a report names the probe among what failed: one key, the same on every report, so the
|
||||||
|
// mesh counts a package manager that keeps not answering as one failure that goes on.
|
||||||
|
func (u *unanswered) failedKey() string { return "the package manager (" + u.tool + ")" }
|
||||||
|
|
||||||
|
// said is the line the apply says before it begins.
|
||||||
|
func (u *unanswered) said() string {
|
||||||
|
return u.tool + " does not answer here, so every package in this declaration fails and the rest of it is " +
|
||||||
|
"applied (novox/hq issue 233): " + u.words
|
||||||
|
}
|
||||||
|
|
||||||
|
// partial is the failure of an apply that applied everything else, when no package was declared to fail.
|
||||||
|
func (u *unanswered) partial() error {
|
||||||
|
return errors.New("applied only in part: " + u.words + "\n\nevery resource that does not need " + u.tool +
|
||||||
|
" was attempted; none here needed it, and " + u.tool + " must answer before this machine applies whole " +
|
||||||
|
"(novox/hq issue 233)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// probePackageManager proves the machine is the system the host was built for, before anything is applied.
|
||||||
|
//
|
||||||
|
// **A package manager that does not answer fails only what needs it** (novox/hq issue 233). The probe was a
|
||||||
|
// refusal of the whole declaration — right for a host installed on a machine it was not built for, and a
|
||||||
|
// wedge when what the probe reads is the mesh's own: a declaration that left the package manager's module
|
||||||
|
// out removed its configuration file, and from then on every declaration was refused, the one that would
|
||||||
|
// write the file back and the node-engine's own upgrade included. A person on the machine was the only way
|
||||||
|
// back. Now the rest applies — files, units, accounts, the node-engine itself — and each package fails with
|
||||||
|
// the probe's words (the system's PackageInstalled asks the same probe first), so a later declaration can
|
||||||
|
// undo what an earlier one did. Before the rest is applied, a declaration composed from an older view of the
|
||||||
|
// assignments than this machine applied has already been refused (issue 234's generation).
|
||||||
|
//
|
||||||
|
// **A package manager that is not there at all still refuses whole.** That is the host installed on a
|
||||||
|
// machine it was not built for — the arch host on Alpine — which is what the probe exists to catch, and no
|
||||||
|
// declaration the mesh sends removes a package manager's program. Only one that is there and errors (a
|
||||||
|
// configuration file gone, a database locked) is the mesh's own fault and applies in part.
|
||||||
|
//
|
||||||
|
// A system with no package manager (android) still refuses whole: what its probe asks is the system itself,
|
||||||
|
// which no resource the mesh declares can remove, and nothing else stands between it and the wrong machine.
|
||||||
|
func probePackageManager(ctx context.Context, sys system.System) (*unanswered, error) {
|
||||||
|
err := sys.Confirm(ctx, apply.ExecRunner)
|
||||||
|
if err == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
tool, ok := packageManagers[sys.Name()]
|
||||||
|
if !ok || errors.Is(err, exec.ErrNotFound) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &unanswered{tool: tool, words: err.Error()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyPass is one apply of the declaration over what the node's state says.
|
||||||
|
type applyPass func(known store.State) (apply.Report, store.State, error)
|
||||||
|
|
||||||
|
// applyMindingProbe applies, and **once more when the package manager did not answer and this apply wrote
|
||||||
|
// something** (novox/hq issue 233). What it wrote may be what the package manager needed — the declaration
|
||||||
|
// that names its module again writes its configuration file back — and then the packages go in now rather
|
||||||
|
// than at the next reconcile. Asked again only then: an apply that wrote nothing changed nothing the package
|
||||||
|
// manager reads. The second pass is the same declaration over the state the first left, so what the first
|
||||||
|
// applied reads unchanged and only what needs the package manager moves.
|
||||||
|
//
|
||||||
|
// It returns the last pass's account, the probe as it stands after it (nil once the package manager
|
||||||
|
// answers), and the first pass's outcomes when a second ran, for whoever counts what an apply moved.
|
||||||
|
func applyMindingProbe(ctx context.Context, sys system.System, probe *unanswered, known store.State,
|
||||||
|
pass applyPass, say func(string)) (apply.Report, store.State, error, *unanswered, []apply.Outcome) {
|
||||||
|
report, updated, err := pass(known)
|
||||||
|
if probe == nil || !wrote(report, err) {
|
||||||
|
return report, updated, err, probe, nil
|
||||||
|
}
|
||||||
|
again, refused := probePackageManager(ctx, sys)
|
||||||
|
if refused != nil || again != nil {
|
||||||
|
return report, updated, err, probe, nil
|
||||||
|
}
|
||||||
|
say(probe.tool + " answers now that this apply wrote what it reads; applying again, so its packages are " +
|
||||||
|
"installed in this apply (novox/hq issue 233)")
|
||||||
|
first := append([]apply.Outcome(nil), report.Outcomes...)
|
||||||
|
var partial *apply.Error
|
||||||
|
if errors.As(err, &partial) {
|
||||||
|
first = append(first, partial.Done.Outcomes...)
|
||||||
|
}
|
||||||
|
report, updated, err = pass(updated)
|
||||||
|
return report, updated, err, nil, first
|
||||||
|
}
|
||||||
|
|
||||||
|
// wrote is whether an apply moved anything on the machine, failed or not.
|
||||||
|
func wrote(report apply.Report, err error) bool {
|
||||||
|
if report.Changed() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var partial *apply.Error
|
||||||
|
return errors.As(err, &partial) && partial.Done.Changed()
|
||||||
|
}
|
||||||
|
|
||||||
|
// unapplied is which declared resources an apply did not apply, failed or not attempted, sorted; nil for a
|
||||||
|
// clean one.
|
||||||
|
func unapplied(d *declaration.Declaration, report apply.Report, err error) []string {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
done := map[string]bool{}
|
||||||
|
for _, o := range report.Outcomes {
|
||||||
|
done[o.ID] = o.Action != "skipped"
|
||||||
|
}
|
||||||
|
var partial *apply.Error
|
||||||
|
if errors.As(err, &partial) {
|
||||||
|
for _, o := range partial.Done.Outcomes {
|
||||||
|
done[o.ID] = done[o.ID] || o.Action != "skipped"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if !done[r.Identity()] {
|
||||||
|
out = append(out, r.Identity())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
if len(out) == 0 {
|
||||||
|
// Failed, and every resource has an account: what failed is no resource (the firewall found, a removal).
|
||||||
|
out = []string{"(apply)"}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withUnanswered adds the probe to what a report says failed.
|
||||||
|
func withUnanswered(failed map[string]string, u *unanswered) map[string]string {
|
||||||
|
if u == nil {
|
||||||
|
return failed
|
||||||
|
}
|
||||||
|
if failed == nil {
|
||||||
|
failed = map[string]string{}
|
||||||
|
}
|
||||||
|
failed[u.failedKey()] = u.said()
|
||||||
|
return failed
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq issue 233: a package manager that does not answer is said even when nothing declared
|
||||||
|
// needs it — the apply is partial, never "applied" — and the report names it under one key, so the mesh
|
||||||
|
// counts it as one failure that goes on.
|
||||||
|
func TestAPackageManagerThatDoesNotAnswerIsSaidWithNoPackageDeclared(t *testing.T) {
|
||||||
|
unansweringPacman(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||||
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||||
|
`","content":"[options]\n"}]}`)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "applied only in part") || !strings.Contains(err.Error(), "could not be read") {
|
||||||
|
t.Fatalf("an apply whose package manager did not answer was not said to be partial: %v", err)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(conf); statErr != nil {
|
||||||
|
t.Fatalf("the file was not written: %v", statErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||||
|
if report.Refused != "" {
|
||||||
|
t.Fatalf("refused whole: %s", report.Refused)
|
||||||
|
}
|
||||||
|
why, ok := report.Failed["the package manager (pacman)"]
|
||||||
|
if !ok || !strings.Contains(why, "could not be read") {
|
||||||
|
t.Errorf("the report does not name the package manager with the probe's words: %v", report.Failed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A system whose probe asks the system itself, not a tool a resource needs, still refuses whole: nothing
|
||||||
|
// the mesh declares can remove it, and nothing else stands between that host and the wrong machine.
|
||||||
|
func TestASystemWithNoPackageManagerStillRefusesWhole(t *testing.T) {
|
||||||
|
t.Setenv("PATH", t.TempDir())
|
||||||
|
sys, err := system.For("android")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
probe, err := probePackageManager(context.Background(), sys)
|
||||||
|
if err == nil || probe != nil {
|
||||||
|
t.Fatalf("the android host's failed probe did not refuse: %v, %+v", err, probe)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pacmanAnsweringOnce is a pacman that does not answer until conf exists, as the workstation's did until its
|
||||||
|
// configuration file was written back, and installs into a directory of its own once it does.
|
||||||
|
func pacmanAnsweringOnce(t *testing.T, conf string) {
|
||||||
|
t.Helper()
|
||||||
|
bin, installed := t.TempDir(), t.TempDir()
|
||||||
|
script := "#!/bin/sh\n" +
|
||||||
|
"[ -f '" + conf + "' ] || { echo \"error: config file " + conf + " could not be read: No such file or directory\" >&2; exit 1; }\n" +
|
||||||
|
"for a; do last=$a; done\n" +
|
||||||
|
"case \"$1\" in\n" +
|
||||||
|
"-Q) [ \"$2\" = pacman ] && exit 0; [ -f '" + installed + "'/\"$2\" ] && exit 0; exit 1;;\n" +
|
||||||
|
"-S) touch '" + installed + "'/\"$last\"; exit 0;;\n" +
|
||||||
|
"esac\nexit 1\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends the review of novox/hq issue 233: **the apply that writes the package manager's configuration back
|
||||||
|
// installs its packages too**. The probe is asked again once the partial apply wrote something, and when it
|
||||||
|
// answers the declaration is applied again over what the first pass left — the packages in this apply, not five
|
||||||
|
// minutes later, and the report clean.
|
||||||
|
func TestThePackagesGoInWhenTheApplyThatRestoresThePackageManagerWroteIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
|
pacmanAnsweringOnce(t, conf)
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
|
// The file first, as the pacman module declares it: a module's apply stops at its first failed resource
|
||||||
|
// (design 42 zfs-owns-zfs §1, answer A-26), so the file that makes the package manager answer comes before the
|
||||||
|
// package that needs it.
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||||
|
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||||
|
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||||
|
|
||||||
|
if report.Refused != "" || len(report.Failed) != 0 {
|
||||||
|
t.Fatalf("the apply that made the package manager answer did not install its packages: refused %q, failed %v",
|
||||||
|
report.Refused, report.Failed)
|
||||||
|
}
|
||||||
|
if !slices.Contains(report.Applied, "pacman.base") || !slices.Contains(report.Applied, "pacman.conf") {
|
||||||
|
t.Errorf("the report does not account for both: %v", report.Applied)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends decision (1) of the review of novox/hq issue 233: **a package manager that is not there at all still
|
||||||
|
// refuses whole**. That is the host installed on the wrong machine, which the probe exists to catch.
|
||||||
|
func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) {
|
||||||
|
t.Setenv("PATH", t.TempDir())
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
dir := t.TempDir()
|
||||||
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||||
|
`","content":"[options]\n"}]}`)
|
||||||
|
|
||||||
|
report := applyAndKeep(context.Background(), options{state: filepath.Join(dir, "state.json")}, raw, nil, nil, nil)
|
||||||
|
|
||||||
|
if !strings.Contains(report.Refused, "pacman does not answer") {
|
||||||
|
t.Fatalf("a machine with no pacman at all was not refused whole: %+v", report)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(conf); err == nil {
|
||||||
|
t.Error("the refused declaration touched the machine")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write
|
||||||
|
// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:<provision>}`, and the node-engine
|
||||||
|
// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the
|
||||||
|
// placeholder alone.
|
||||||
|
//
|
||||||
|
// A placeholder names the credential as the module calls it, `${secret:<name>}` (catalogue.SecretLocal, ADR 0094): the
|
||||||
|
// local name where the module keeps several values of one provision, else the provision. For each, it finds the one
|
||||||
|
// provision key `<node>.<consumer>.<provision>[.<local>]` of the seat mesh-provisioner's state `provisions` (clause 1)
|
||||||
|
// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair
|
||||||
|
// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client,
|
||||||
|
// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the
|
||||||
|
// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot
|
||||||
|
// connect.
|
||||||
|
type provisionSecrets struct {
|
||||||
|
node string
|
||||||
|
// list is every key of mesh-provisioner's `provisions` under prefix (`<node>.<consumer>.`), by key: what this
|
||||||
|
// engine may read, its own node's (`<node>.>`).
|
||||||
|
list func(ctx context.Context, prefix string) (map[string][]byte, error)
|
||||||
|
// own asks the vault's verb `own` in this engine's name.
|
||||||
|
own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
|
||||||
|
// open opens a value sealed to this machine.
|
||||||
|
open func(sealed string) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretPlaceholder is `${secret:<name>}`.
|
||||||
|
var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`)
|
||||||
|
|
||||||
|
// fill is content with each provision placeholder of module's filled, and whether any was.
|
||||||
|
func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) {
|
||||||
|
found := secretPlaceholder.FindAllSubmatch(content, -1)
|
||||||
|
if len(found) == 0 {
|
||||||
|
return content, false, nil
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found))
|
||||||
|
}
|
||||||
|
if p.list == nil || p.own == nil || p.open == nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module)
|
||||||
|
}
|
||||||
|
prefix := p.node + "." + module + "."
|
||||||
|
keys, err := p.list(ctx, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err)
|
||||||
|
}
|
||||||
|
out := string(content)
|
||||||
|
done := map[string]bool{}
|
||||||
|
for _, f := range found {
|
||||||
|
whole, name := string(f[0]), string(f[1])
|
||||||
|
if done[whole] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
done[whole] = true
|
||||||
|
pr, key, err := byName(keys, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err)
|
||||||
|
}
|
||||||
|
value, err := p.value(ctx, key, pr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
out = strings.ReplaceAll(out, whole, string(value))
|
||||||
|
}
|
||||||
|
return []byte(out), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key.
|
||||||
|
func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) {
|
||||||
|
var hit []string
|
||||||
|
var got provisions.Provision
|
||||||
|
for key, raw := range keys {
|
||||||
|
pr, err := provisions.Decode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if catalogue.SecretLocal(pr.Provision, pr.Local) == name {
|
||||||
|
hit, got = append(hit, key), pr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(hit) {
|
||||||
|
case 0:
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", buckets.ProvisionsRead)
|
||||||
|
case 1:
|
||||||
|
return got, hit[0], nil
|
||||||
|
}
|
||||||
|
sort.Strings(hit)
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// value is one provision's value on this node, opened.
|
||||||
|
func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) {
|
||||||
|
if pr.State != provisions.Provided || pr.Reference == nil {
|
||||||
|
why := pr.State
|
||||||
|
if pr.Why != "" {
|
||||||
|
why += ": " + pr.Why
|
||||||
|
}
|
||||||
|
if pr.State == provisions.Provided {
|
||||||
|
why = "provided with no reference to the vault's value"
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why)
|
||||||
|
}
|
||||||
|
// The reference names a credential of this provision, as the vault names it (provisions.VaultName).
|
||||||
|
ours := false
|
||||||
|
for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} {
|
||||||
|
ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID
|
||||||
|
}
|
||||||
|
if !ours {
|
||||||
|
return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID)
|
||||||
|
}
|
||||||
|
module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local
|
||||||
|
type asked struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Provision string `json:"provision"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
ConsumerModule string `json:"consumer_module"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
Local string `json:"local,omitempty"`
|
||||||
|
}
|
||||||
|
ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local}
|
||||||
|
answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s: %w", key, err)
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Credentials []struct {
|
||||||
|
asked
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
|
} `json:"credentials"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer, &a); err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err)
|
||||||
|
}
|
||||||
|
if a.Node != p.node {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node)
|
||||||
|
}
|
||||||
|
for _, c := range a.Credentials {
|
||||||
|
if c.asked != ask {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The value the provision names, by the vault's credential name and its generation (clause 6): another is refused.
|
||||||
|
if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+
|
||||||
|
"nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation)
|
||||||
|
}
|
||||||
|
value, err := p.open(string(c.Sealed))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the vault answered no value of %s for this machine", key)
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pullsOf is the images an apply could not find or pull, each with the seat it came through and that seat's holder as the
|
||||||
|
// declaration names them beside the container (design 51 *distribution's first delivery*, ENG-8, ENG-10). An image the
|
||||||
|
// declaration names no holder for (a public registry's) carries none: its pull is not tried again on a store settling.
|
||||||
|
func pullsOf(d *declaration.Declaration, err error) []agent.PullFailure {
|
||||||
|
var out []agent.PullFailure
|
||||||
|
for _, p := range apply.PullErrors(err) {
|
||||||
|
f := agent.PullFailure{Image: p.Image, Started: p.Started}
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if c, ok := r.(*declaration.Container); ok && c.Image == p.Image && c.ImageHolder != "" {
|
||||||
|
f.Seat, f.Holder = c.ImageSeat, c.ImageHolder
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The record check reads the engine's applied state: a composition one of whose resources has no record there is not
|
||||||
|
// recorded (issue 1945).
|
||||||
|
func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
var s store.State
|
||||||
|
s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
|
||||||
|
if err := store.Save(path, s); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c := engcomp.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
|
||||||
|
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
|
||||||
|
recorded := recordedIn(path, func(string) {})
|
||||||
|
if !recorded(context.Background(), c) {
|
||||||
|
t.Fatal("a composition whose every resource has a record was not recorded")
|
||||||
|
}
|
||||||
|
c.Resources = append(c.Resources, json.RawMessage(
|
||||||
|
`{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`))
|
||||||
|
if recorded(context.Background(), c) {
|
||||||
|
t.Fatal("a composition with a resource the state holds no record of was recorded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
||||||
|
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||||
|
t.Run("1957: the machine shows a resource not active", testTheMachineShowsAResourceNotActive)
|
||||||
|
st := liveness.Statement{Resources: []liveness.State{
|
||||||
|
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
||||||
|
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
||||||
|
{Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true},
|
||||||
|
{Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy},
|
||||||
|
}}
|
||||||
|
if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" {
|
||||||
|
t.Fatalf("applied again: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module is applied again only for a resource newly missing: one still missing at the next change of the statement is
|
||||||
|
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
||||||
|
// that comes back and goes again is newly missing again (the review of issue 1945).
|
||||||
|
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
||||||
|
t.Run("1957: a resource newly down is an edge", testOnlyAResourceNewlyDownIsAnEdge)
|
||||||
|
missing := func(ids ...string) liveness.Statement {
|
||||||
|
var st liveness.Statement
|
||||||
|
for _, id := range ids {
|
||||||
|
st.Resources = append(st.Resources, liveness.State{Resource: liveness.Resource{Module: "matrix", ID: id},
|
||||||
|
State: liveness.Unhealthy, Missing: true})
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
var last map[string]bool
|
||||||
|
var got []string
|
||||||
|
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("first missing: %v", got)
|
||||||
|
}
|
||||||
|
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("still missing was acted on again: %v", got)
|
||||||
|
}
|
||||||
|
got, last = newlyMissing(last, missing())
|
||||||
|
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("missing again after it came back: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file the provisioner client fills is left to it by the apply and has no record until the client writes it: it does
|
||||||
|
// not make its composition unrecorded (the review of issue 1945).
|
||||||
|
func TestAFilledFileDoesNotMakeItsCompositionUnrecorded(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
var s store.State
|
||||||
|
s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
|
||||||
|
if err := store.Save(path, s); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c := engcomp.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{
|
||||||
|
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`),
|
||||||
|
json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}}
|
||||||
|
if !recordedIn(path, func(string) {})(context.Background(), c) {
|
||||||
|
t.Fatal("a filled file the client has not written made its composition unrecorded")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296,
|
||||||
|
// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the
|
||||||
|
// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool.
|
||||||
|
func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery {
|
||||||
|
ask := func(ctx context.Context, verb string, in, out any) error {
|
||||||
|
raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(raw, out)
|
||||||
|
}
|
||||||
|
return &agent.Recovery{
|
||||||
|
FormChanges: composer.FormChanges,
|
||||||
|
SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a)
|
||||||
|
},
|
||||||
|
RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a)
|
||||||
|
},
|
||||||
|
KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) {
|
||||||
|
return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a)
|
||||||
|
},
|
||||||
|
Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) {
|
||||||
|
heard, stop, err := session.HearRecoveryEnding(kind, module, kept)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
out := make(chan agent.Ending, 64)
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case ev := <-heard:
|
||||||
|
select {
|
||||||
|
case out <- agent.Ending(ev):
|
||||||
|
case <-done:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-done:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return out, func() { stop(); close(done) }, nil
|
||||||
|
},
|
||||||
|
Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) {
|
||||||
|
return session.KeepBound(ctx, what, id, module, since, d)
|
||||||
|
},
|
||||||
|
CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error {
|
||||||
|
return session.CallTool(ctx, module, c.Tool, c.Arguments)
|
||||||
|
},
|
||||||
|
WaitMinutes: recoveryWaitMinutes(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the
|
||||||
|
// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71).
|
||||||
|
var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes"
|
||||||
|
|
||||||
|
// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the
|
||||||
|
// file is missing, empty or not a positive number (design 47 G §554, A-112).
|
||||||
|
func recoveryWaitMinutes() int {
|
||||||
|
raw, err := os.ReadFile(recoveryWaitFile)
|
||||||
|
if err != nil {
|
||||||
|
return agent.RecoveryWaitMinutes
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return agent.RecoveryWaitMinutes
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
@@ -0,0 +1,301 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||||
|
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
|
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/nodegrant"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears
|
||||||
|
// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's
|
||||||
|
// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and
|
||||||
|
// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant.
|
||||||
|
func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) {
|
||||||
|
perms := nodegrant.For(itNode)
|
||||||
|
perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode,
|
||||||
|
"mesh.mod.db.tool.rebuild."+itNode)
|
||||||
|
perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>")
|
||||||
|
b := newItBedAs(t, perms)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll},
|
||||||
|
{meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} {
|
||||||
|
if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
js, err := jetstream.New(b.conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
say := func(act, outcome string) {
|
||||||
|
raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()})
|
||||||
|
if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var restoreAsked meshrecovery.RestoreKeptInput
|
||||||
|
// The kept points the stand-in answers: the one of the first checks, then (below) what the agent's revert finds, and none.
|
||||||
|
var pointsMu sync.Mutex
|
||||||
|
points := []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1",
|
||||||
|
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}
|
||||||
|
// The restore the agent's revert asks waits for the stand-in to say it restored: gated until released.
|
||||||
|
var gated atomic.Bool
|
||||||
|
release := make(chan struct{})
|
||||||
|
var restores atomic.Int32
|
||||||
|
for verb, answer := range map[string]func(m *nats.Msg) any{
|
||||||
|
meshrecovery.VerbKeptPoints: func(*nats.Msg) any {
|
||||||
|
pointsMu.Lock()
|
||||||
|
defer pointsMu.Unlock()
|
||||||
|
return meshrecovery.KeptPointsAnswer{Points: points}
|
||||||
|
},
|
||||||
|
meshrecovery.VerbRestoreKept: func(m *nats.Msg) any {
|
||||||
|
pointsMu.Lock()
|
||||||
|
_ = json.Unmarshal(m.Data, &restoreAsked)
|
||||||
|
pointsMu.Unlock()
|
||||||
|
act := fmt.Sprintf("r%d", restores.Add(1))
|
||||||
|
if gated.Load() {
|
||||||
|
go func() { <-release; say(act, meshrecovery.RestoredEventOutcomeDone) }()
|
||||||
|
} else {
|
||||||
|
say(act, meshrecovery.RestoredEventOutcomeDone)
|
||||||
|
}
|
||||||
|
return meshrecovery.RestoreKeptAnswer{Accepted: act}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
answer := answer
|
||||||
|
sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) {
|
||||||
|
out, _ := json.Marshal(map[string]any{"result": answer(m)})
|
||||||
|
_ = m.Respond(out)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
}
|
||||||
|
var toolArgs map[string]any
|
||||||
|
var toolCalls atomic.Int32
|
||||||
|
toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) {
|
||||||
|
var args map[string]any
|
||||||
|
_ = json.Unmarshal(m.Data, &args)
|
||||||
|
pointsMu.Lock()
|
||||||
|
toolArgs = args
|
||||||
|
pointsMu.Unlock()
|
||||||
|
toolCalls.Add(1)
|
||||||
|
_ = m.Respond([]byte(`{"result":{}}`))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = toolSub.Unsubscribe() })
|
||||||
|
|
||||||
|
// A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number.
|
||||||
|
for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} {
|
||||||
|
recoveryWaitFile = filepath.Join(t.TempDir(), "w")
|
||||||
|
if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recoveryWaitMinutes(); got != want {
|
||||||
|
t.Errorf("the file %q gives %d minutes, want %d", content, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
recoveryWaitFile = filepath.Join(t.TempDir(), "missing")
|
||||||
|
if got := recoveryWaitMinutes(); got != 90 {
|
||||||
|
t.Errorf("no file gives %d minutes, want 90", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject
|
||||||
|
session := b.startSession()
|
||||||
|
r := recoveryEnv(session, nil)
|
||||||
|
|
||||||
|
events, stop, err := r.Hear(ctx, "restored", "db", "cache")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
if first := <-events; first.Act != "old" || first.Outcome != "failed" {
|
||||||
|
t.Fatalf("the first event heard is %+v, want the last one said before", first)
|
||||||
|
}
|
||||||
|
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"})
|
||||||
|
if err != nil || len(pts.Points) != 1 {
|
||||||
|
t.Fatalf("kept-points: %+v, %v", pts, err)
|
||||||
|
}
|
||||||
|
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"})
|
||||||
|
pointsMu.Lock()
|
||||||
|
firstAsked := restoreAsked
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if err != nil || got.Accepted != "r1" || firstAsked.Snapshot != "snap-1" {
|
||||||
|
t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, firstAsked)
|
||||||
|
}
|
||||||
|
if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" {
|
||||||
|
t.Fatalf("the ending heard is %+v", ev)
|
||||||
|
}
|
||||||
|
err = r.CatchUp(ctx, "db", *pts.Points[0].CatchUp)
|
||||||
|
pointsMu.Lock()
|
||||||
|
gotArgs := toolArgs
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if err != nil || gotArgs["all"] != true {
|
||||||
|
t.Fatalf("catch-up: %v, arguments %v", err, gotArgs)
|
||||||
|
}
|
||||||
|
expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer end()
|
||||||
|
select {
|
||||||
|
case <-expired:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("the bound's end, the bus's marker, was not heard")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Design 47 G §296, §297, §554, §560, A-113: the node agent whole drives a revert want of db through the same recovery wiring
|
||||||
|
// (recoveryEnv over the session and the node's composer): db runs at a build whose cache is of form 2, and its want goes back to a
|
||||||
|
// build of form 1. With no point of form 1 taken before the newer build, its have says the restore point is missing, naming it,
|
||||||
|
// and the newer build runs again. With the point, the have is held (backing up) until the stand-in says restored, then the
|
||||||
|
// older build is applied and the point's catch-up tool is called.
|
||||||
|
dbManifest := func(form int, content string) string {
|
||||||
|
return `{"module":"db","version":"1","state":[{"name":"cache","form":` + fmt.Sprint(form) + `}],"resources":[{"id":"conf","type":"file","path":"` +
|
||||||
|
filepath.Join(b.dir, "db.conf") + `","mode":"0644","content":"` + content + `\n"}]}`
|
||||||
|
}
|
||||||
|
fpOld, fpNew := fpIt("a"), fpIt("b")
|
||||||
|
b.registerEntry("db", fpOld, dbManifest(1, "v1"))
|
||||||
|
b.registerEntry("db", fpNew, dbManifest(2, "v2"))
|
||||||
|
for _, name := range []string{buckets.Manifests, selfcompose.ResolvedBucket, buckets.ProposalCandidates, link.PackagesBucket} {
|
||||||
|
if _, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 5}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assignment, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: buckets.Assignments, History: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "db"}}})
|
||||||
|
hkey, _ := nec.HaveNodeKey(itNode)
|
||||||
|
rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}})
|
||||||
|
if _, err := b.haves.Put(hkey, rec); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for key, value := range places {
|
||||||
|
if _, err := assignment.Put(key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.want("db", fpNew, "", "")
|
||||||
|
stopAgent := b.start()
|
||||||
|
defer stopAgent()
|
||||||
|
within(t, 20*time.Second, "db does not run its newer build", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Fp == fpNew && h.Phase == nec.PhaseRunning && b.file("db") == "v2\n"
|
||||||
|
})
|
||||||
|
// The engine says what runs, as it does after each apply: the composer keeps the entry of the running build while the want moves on.
|
||||||
|
b.running(itNode, "db", fpNew, meshcatalogclient.ManifestDigest([]byte(dbManifest(2, "v2"))))
|
||||||
|
// The stand-in holds no point of form 1: the revert is held, the have names the point missing, the newer build runs.
|
||||||
|
pointsMu.Lock()
|
||||||
|
points = nil
|
||||||
|
pointsMu.Unlock()
|
||||||
|
b.want("db", fpOld, "", "")
|
||||||
|
within(t, 20*time.Second, "the have does not say the restore point is missing", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Phase == nec.PhaseHeld && strings.Contains(h.Detail, "restore point missing") &&
|
||||||
|
strings.Contains(h.Detail, "db's cache has none of form 1 taken before "+fpNew)
|
||||||
|
})
|
||||||
|
if b.have("db").Fp != fpNew || b.file("db") != "v2\n" || restores.Load() != 1 {
|
||||||
|
t.Fatalf("the missing point left db %+v with %q, %d restore(s) asked", b.have("db"), b.file("db"), restores.Load())
|
||||||
|
}
|
||||||
|
// The stand-in now holds the point: the want answered "again" asks the restore, and the have waits for its ending.
|
||||||
|
pointsMu.Lock()
|
||||||
|
points = []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fpNew, Snapshot: "snap-2", At: time.Now(),
|
||||||
|
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": "again"}}}}
|
||||||
|
pointsMu.Unlock()
|
||||||
|
gated.Store(true)
|
||||||
|
calls := toolCalls.Load()
|
||||||
|
b.want("db", fpOld, "", meshwants.WantsByWantNodeModuleValueBackupAgain)
|
||||||
|
within(t, 20*time.Second, "the restore of the revert is not asked", func() bool { return restores.Load() == 2 })
|
||||||
|
within(t, 20*time.Second, "the have does not wait for the restore", func() bool { return b.have("db").Phase == nec.PhaseBackingUp })
|
||||||
|
// The restored event is held until close(release): the older build must not be applied before the stand-in says restored.
|
||||||
|
if h := b.have("db"); h.Phase != nec.PhaseBackingUp || b.file("db") != "v2\n" || toolCalls.Load() != calls {
|
||||||
|
t.Fatalf("before the stand-in says restored the have is %+v, the file %q, %d catch-up call(s) more", h, b.file("db"), toolCalls.Load()-calls)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
within(t, 20*time.Second, "the older build is not applied after the restore", func() bool {
|
||||||
|
h := b.have("db")
|
||||||
|
return h.Fp == fpOld && h.Phase == nec.PhaseRunning && b.file("db") == "v1\n"
|
||||||
|
})
|
||||||
|
within(t, 20*time.Second, "the catch-up tool is not called after the restore", func() bool { return toolCalls.Load() == calls+1 })
|
||||||
|
pointsMu.Lock()
|
||||||
|
asked, gotArgs := restoreAsked, toolArgs
|
||||||
|
pointsMu.Unlock()
|
||||||
|
if asked.Snapshot != "snap-2" || asked.Fingerprint != fpNew || gotArgs["all"] != "again" {
|
||||||
|
t.Errorf("the revert asked %+v and called the catch-up with %v", asked, gotArgs)
|
||||||
|
}
|
||||||
|
stopAgent()
|
||||||
|
|
||||||
|
// Design 47 G A-293: over the same session, the builder's build ends the composer follows: the stream read whole first (the
|
||||||
|
// last end of each module and fingerprint, in the stream's order), then the read's end, then each end said after it.
|
||||||
|
if _, err := b.js.AddStream(&nats.StreamConfig{Name: link.BuildEndsStream, Subjects: []string{link.BuildEndsSubject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ended := func(fp, outcome, why string) {
|
||||||
|
raw, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: fp, Outcome: outcome, Error: why})
|
||||||
|
if _, err := b.js.Publish("mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("db", fp), raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ended("f1", builds.BuildFailed, "first")
|
||||||
|
ended("f1", builds.BuildFailed, "no go")
|
||||||
|
ended("f0", builds.BuildDone, "")
|
||||||
|
heard := make(chan string, 8)
|
||||||
|
follow, stopFollow := context.WithCancel(ctx)
|
||||||
|
defer stopFollow()
|
||||||
|
go func() {
|
||||||
|
_ = session.FollowBuildEnds(follow, func(e builds.BuildEnded) { heard <- e.Fingerprint + ":" + e.Outcome + ":" + e.Error },
|
||||||
|
func() { heard <- "read" })
|
||||||
|
}()
|
||||||
|
for i, want := range []string{"f1:failed:no go", "f0:done:", "read", "f2:failed:later"} {
|
||||||
|
if want == "f2:failed:later" {
|
||||||
|
ended("f2", builds.BuildFailed, "later")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case got := <-heard:
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("build end %d heard %q, want %q", i, got, want)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("build end %d, %q, was not heard", i, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSession is a session on the node's own connection, with no node agent run on it.
|
||||||
|
func (b *itBed) startSession() *link.Session {
|
||||||
|
b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||||
|
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||||
|
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||||
|
return b.session
|
||||||
|
}
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/ed25519"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/tunnel"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
|
|
||||||
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
|
|
||||||
// proof signed by the identity key, over the previous key, the new one and the tunnel.
|
|
||||||
|
|
||||||
func anEnrolledNode(t *testing.T) identity.Identity {
|
|
||||||
t.Helper()
|
|
||||||
mine, err := identity.Generate("anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Overlay, err = identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
|
|
||||||
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
|
|
||||||
return mine
|
|
||||||
}
|
|
||||||
|
|
||||||
func aFoundTunnel(t *testing.T) tunnel.Found {
|
|
||||||
t.Helper()
|
|
||||||
private, err := identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
|
|
||||||
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
|
|
||||||
mine := anEnrolledNode(t)
|
|
||||||
found := aFoundTunnel(t)
|
|
||||||
before := mine.Overlay.Public
|
|
||||||
|
|
||||||
taken, rekey, err := rekeyOnto(mine, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
|
|
||||||
t.Fatal("the overlay key is not the tunnel's")
|
|
||||||
}
|
|
||||||
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
|
|
||||||
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
|
|
||||||
taken.Membership.Broker != mine.Membership.Broker {
|
|
||||||
t.Fatal("something other than the overlay key moved")
|
|
||||||
}
|
|
||||||
if taken.OverlayBefore != before {
|
|
||||||
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
|
|
||||||
}
|
|
||||||
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
|
|
||||||
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
|
|
||||||
t.Fatalf("the rekey does not say what moved: %+v", rekey)
|
|
||||||
}
|
|
||||||
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the rekey is not signed by this node's identity key over what it says")
|
|
||||||
}
|
|
||||||
if ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the proof is not bound to the node")
|
|
||||||
}
|
|
||||||
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
|
|
||||||
if strings.Contains(said, found.PrivateKey()) {
|
|
||||||
t.Fatal("the private key travels")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
|
|
||||||
// names is still the one before the take, so the mesh can tell a repeat from a replay.
|
|
||||||
again, second, err := rekeyOnto(taken, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
|
|
||||||
t.Fatalf("a take run again does not name the key before the first: %+v", second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// unansweringPacman puts a pacman on PATH that answers as one whose configuration file is gone, as the
|
||||||
|
// workstation's did on 2026-10-04 (novox/hq issue 233), and makes this an arch host.
|
||||||
|
func unansweringPacman(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
bin := t.TempDir()
|
||||||
|
script := "#!/bin/sh\necho 'error: config file /etc/pacman.conf could not be read: No such file or directory' >&2\nexit 1\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReplay233 replays novox/hq issue 233. The package manager's module wrote pacman's configuration
|
||||||
|
// file; a declaration that left the module out removed it; from then on pacman did not answer, and the
|
||||||
|
// node-engine refused every declaration whole — the one that named the module again and would have
|
||||||
|
// written the file back included. A person restored the file by hand an hour and a half later.
|
||||||
|
//
|
||||||
|
// Held to: a declaration that restores the file is applied while pacman does not answer. Only what needs
|
||||||
|
// pacman fails, with the probe's words; the file is written and recorded. Written only with what the
|
||||||
|
// node-engine had before the fix, so it lays over the commit before it, where it fails.
|
||||||
|
func TestReplay233(t *testing.T) {
|
||||||
|
unansweringPacman(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||||
|
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||||
|
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||||
|
|
||||||
|
if report.Refused != "" {
|
||||||
|
t.Fatalf("the declaration that restores the package manager's configuration was refused whole: %s", report.Refused)
|
||||||
|
}
|
||||||
|
if got, err := os.ReadFile(conf); err != nil || string(got) != "[options]\n" {
|
||||||
|
t.Fatalf("the file that restores the package manager was not written: %q, %v", got, err)
|
||||||
|
}
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, ok := known.Find("pacman.conf"); !ok {
|
||||||
|
t.Error("the file written was not recorded, so the next apply would not know it is the mesh's")
|
||||||
|
}
|
||||||
|
words := strings.Join(failedWords(report.Failed), "\n")
|
||||||
|
if !strings.Contains(words, "pacman does not answer") || !strings.Contains(words, "could not be read") {
|
||||||
|
t.Errorf("the report does not say the package manager did not answer, in the probe's words: %q", words)
|
||||||
|
}
|
||||||
|
if !strings.Contains(words, "pacman.base") {
|
||||||
|
t.Errorf("the report does not name the package that was not applied: %q", words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func failedWords(failed map[string]string) []string {
|
||||||
|
var out []string
|
||||||
|
for id, why := range failed {
|
||||||
|
out = append(out, id+": "+why)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 233 at the machine: `mesh-host apply` applies what does not need the package
|
||||||
|
// manager, and says the apply was partial with the probe's words — a non-zero exit, never "applied".
|
||||||
|
func TestApplyByHandAppliesWhatDoesNotNeedAPackageManagerThatDoesNotAnswer(t *testing.T) {
|
||||||
|
unansweringPacman(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
out := &bytes.Buffer{}
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json"), out: out}
|
||||||
|
conf := filepath.Join(dir, "pacman.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` +
|
||||||
|
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||||
|
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||||
|
|
||||||
|
if _, statErr := os.Stat(conf); statErr != nil {
|
||||||
|
t.Fatalf("the file was not written: %v (the apply said %v)", statErr, err)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an apply whose package manager did not answer said it applied")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "pacman does not answer") || !strings.Contains(err.Error(), "pacman.base") {
|
||||||
|
t.Errorf("the failure does not name the package or the probe's words: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "pacman does not answer") {
|
||||||
|
t.Errorf("what was said before applying does not say the package manager does not answer:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The restart count the engine says is its own since it began watching the resource, never since the state
|
||||||
|
// began (novox/hq issue 458).
|
||||||
|
func TestAnUnhealthyResourceSaysWhatItsRestartCountCounts(t *testing.T) {
|
||||||
|
r := liveness.State{Resource: liveness.Resource{ID: "messenger.own-runtime", Kind: "process",
|
||||||
|
Target: "messenger-runtime.service"}, State: liveness.Unhealthy, Reason: liveness.ReasonRestarting, Restarts: 3890}
|
||||||
|
said := unhealthyWords(r)
|
||||||
|
if !strings.Contains(said, "3890 automatic restart(s) since this engine began watching it") ||
|
||||||
|
strings.Contains(said, "counted") {
|
||||||
|
t.Fatalf("the restart count is not said as what it counts: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest
|
||||||
|
// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the
|
||||||
|
// agent has not applied since its start can come from the kept declaration, older than the compositions applied since:
|
||||||
|
// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the
|
||||||
|
// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it
|
||||||
|
// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote,
|
||||||
|
// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is
|
||||||
|
// left as the composition names it.
|
||||||
|
func runsAsWritten(running []engcomp.Composed, statePath string) []engcomp.Composed {
|
||||||
|
if len(running) == 0 || statePath == "" {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
out := make([]engcomp.Composed, len(running))
|
||||||
|
for i, c := range running {
|
||||||
|
out[i] = c
|
||||||
|
var changed []json.RawMessage
|
||||||
|
for j, raw := range c.Resources {
|
||||||
|
if fixed, ok := asWritten(raw, known); ok {
|
||||||
|
if changed == nil {
|
||||||
|
changed = append([]json.RawMessage(nil), c.Resources...)
|
||||||
|
}
|
||||||
|
changed[j] = fixed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if changed != nil {
|
||||||
|
out[i].Resources = changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a
|
||||||
|
// file given by another digest and the file there is what the record says the node wrote.
|
||||||
|
func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) {
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(raw, &fields) != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
var kind, id, path, digest string
|
||||||
|
_ = json.Unmarshal(fields["type"], &kind)
|
||||||
|
_ = json.Unmarshal(fields["id"], &id)
|
||||||
|
_ = json.Unmarshal(fields["path"], &path)
|
||||||
|
_ = json.Unmarshal(fields["digest"], &digest)
|
||||||
|
if kind != string(declaration.TypeFile) || digest == "" || path == "" {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
rec, ok := known.Find(id)
|
||||||
|
if !ok || rec.Wrote == "" || rec.Target != path {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(sum[:]) != rec.Wrote {
|
||||||
|
return nil, false // changed since the node wrote it: not the node's to vouch for
|
||||||
|
}
|
||||||
|
if now := declaration.DigestOf(body); now == digest {
|
||||||
|
return nil, false
|
||||||
|
} else {
|
||||||
|
fields["digest"], _ = json.Marshal(now)
|
||||||
|
}
|
||||||
|
fields["size"], _ = json.Marshal(len(body))
|
||||||
|
fixed, err := json.Marshal(fields)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return fixed, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/views"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/seatwait"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-engine reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636): through the seat mesh-catalog,
|
||||||
|
// from the bootstrapped definitions until it has been read, acting on nothing new meanwhile (the composer's gate), and
|
||||||
|
// following the seat's events. What it holds is given to catalogue at every change, which compiles no seat list.
|
||||||
|
|
||||||
|
// seatListNow is this run's reader, what the node-engine's verbs seat-list and seat-list-again answer from.
|
||||||
|
var seatListNow atomic.Pointer[seatListState]
|
||||||
|
|
||||||
|
// seatListRead answers the verb seat-list: what the engine holds of the list and what it waits for (ADR 0636 §7).
|
||||||
|
func seatListRead() (any, error) {
|
||||||
|
st := seatListNow.Load()
|
||||||
|
if st == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
||||||
|
}
|
||||||
|
status := st.reader.Status()
|
||||||
|
return map[string]any{"status": status, "health": status.Health()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatListAgain answers the verb seat-list-again: the list read now from the seat, and what is held afterwards.
|
||||||
|
func seatListAgain() (any, error) {
|
||||||
|
st := seatListNow.Load()
|
||||||
|
if st == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
||||||
|
}
|
||||||
|
err := st.reader.Reload()
|
||||||
|
st.waiter.Changed() // a read that changed nothing held still ends the wait
|
||||||
|
out, _ := seatListRead()
|
||||||
|
if err != nil {
|
||||||
|
out.(map[string]any)["error"] = err.Error()
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatListState is one run's reader, the finding that it waits, and the channel on which the composer hears a change.
|
||||||
|
type seatListState struct {
|
||||||
|
reader *seatlist.Reader
|
||||||
|
waiter *seatwait.Waiter
|
||||||
|
changed chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sessionBus is the reader's bus over the link: a verb of the seat is asked in the engine's own name, an event heard on
|
||||||
|
// the seat's event subject for as long as ctx lives.
|
||||||
|
type sessionBus struct {
|
||||||
|
ctx context.Context
|
||||||
|
session *link.Session
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b sessionBus) Ask(key string, body any) (json.RawMessage, error) {
|
||||||
|
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||||
|
return b.session.AskMeshSeat(b.ctx, seat, verb, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b sessionBus) Subscribe(pattern string, handle func(string, json.RawMessage) error) error {
|
||||||
|
seat, event, _ := strings.Cut(pattern, ".")
|
||||||
|
heard, stop, err := b.session.Hear(seatdef.EventSubject(seat, event))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-b.ctx.Done():
|
||||||
|
return
|
||||||
|
case body, ok := <-heard:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = handle(pattern, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatWaitClock is the clock of the node's bound; the tests give their own.
|
||||||
|
var seatWaitClock seatwait.Clock = realClock{}
|
||||||
|
|
||||||
|
type realClock struct{}
|
||||||
|
|
||||||
|
func (realClock) Now() time.Time { return time.Now() }
|
||||||
|
func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer {
|
||||||
|
return time.AfterFunc(d, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they
|
||||||
|
// happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions.
|
||||||
|
type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
|
||||||
|
|
||||||
|
// findingsOfTheNode keeps the node's findings through mesh-sdk's held writer (design 47 G §383, §384, §461, A-29): each
|
||||||
|
// finding is written in the node's state `findings` and said as an event on the engine's own stream (a foundation part's
|
||||||
|
// `mesh-host-findings`, §529, A-30), the engine says what it holds at its start and when the keeper asks, and the keeper
|
||||||
|
// reads no state of the engine's.
|
||||||
|
type findingsOfTheNode struct {
|
||||||
|
*nodeFindings
|
||||||
|
}
|
||||||
|
|
||||||
|
// newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on
|
||||||
|
// the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own.
|
||||||
|
func newFindingsOfTheNode(session *link.Session, node, module string, say func(string)) (*findingsOfTheNode, error) {
|
||||||
|
writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
|
||||||
|
Module: module,
|
||||||
|
Node: node,
|
||||||
|
State: link.FindingsState{Session: session},
|
||||||
|
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...),
|
||||||
|
Asked: condseat.SubjectFindingsAsked,
|
||||||
|
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
|
||||||
|
Publish: session.PublishEvent,
|
||||||
|
Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) {
|
||||||
|
return session.ListenFrom(condseat.StreamFindingsAsked, subject, handle)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &findingsOfTheNode{writer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start seeds what the state holds, states the findings now and hears the keeper until ctx ends.
|
||||||
|
func (w *findingsOfTheNode) Start(ctx context.Context) error {
|
||||||
|
go func() { <-ctx.Done(); w.Close() }()
|
||||||
|
return w.nodeFindings.Start(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what
|
||||||
|
// it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what
|
||||||
|
// the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds.
|
||||||
|
func startSeatList(ctx context.Context, session *link.Session, node string, findings *lateFindings, say func(string)) (*seatListState, error) {
|
||||||
|
st := &seatListState{changed: make(chan struct{}, 1)}
|
||||||
|
// Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists.
|
||||||
|
var announce atomic.Pointer[func()]
|
||||||
|
reader, err := seatlist.Start(seatlist.Config{Bus: sessionBus{ctx, session}, Bootstrap: seatlist.Bootstrapped(),
|
||||||
|
OnChange: func(seats []catalogue.Seat) {
|
||||||
|
catalogue.UseSeats(seats)
|
||||||
|
// What the engine reads changed with the list: its record and its event say it (ADR 0618 §4), off the
|
||||||
|
// reader's goroutine, which returns quickly.
|
||||||
|
go session.RestateReads(ctx)
|
||||||
|
if f := announce.Load(); f != nil { // the first read may end before the waiter exists
|
||||||
|
(*f)()
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err)
|
||||||
|
}
|
||||||
|
st.reader = reader
|
||||||
|
st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Bound: viewWaitsBound(),
|
||||||
|
Clock: seatWaitClock, Say: say}
|
||||||
|
told := func() {
|
||||||
|
st.waiter.Changed()
|
||||||
|
select {
|
||||||
|
case st.changed <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
announce.Store(&told)
|
||||||
|
catalogue.UseSeats(reader.Seats())
|
||||||
|
session.RestateReads(ctx) // its first statement holds the bootstrapped seats at least (issue 1966)
|
||||||
|
told()
|
||||||
|
// The first read ends whether the seat answered or not, and a read that held what the reader held already says no
|
||||||
|
// change: its end is told to the waiter and the composer, so a status that moved from reading to heard starts or ends
|
||||||
|
// the bound (ADR 0636 §1). Start's return held nobody's start.
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-reader.Ready():
|
||||||
|
catalogue.UseSeats(reader.Seats())
|
||||||
|
session.RestateReads(ctx)
|
||||||
|
told()
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
seatListNow.Store(st)
|
||||||
|
return st, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lateFindings is the node-engine's finding writer, made when the engine's own module is known (design 47 G §527, §634,
|
||||||
|
// A-149): at once from its applied record, or, on a first start with no such record, when the first composing names it. Till
|
||||||
|
// then a write or a clear is refused in words, never lost in silence.
|
||||||
|
type lateFindings struct {
|
||||||
|
own *link.OwnName
|
||||||
|
make func(module string) (*findingsOfTheNode, error)
|
||||||
|
mu sync.Mutex
|
||||||
|
w *findingsOfTheNode
|
||||||
|
}
|
||||||
|
|
||||||
|
// observer is the module the findings are observed as: the engine's own, "" while it is not known.
|
||||||
|
func (f *lateFindings) observer() string { return f.own.Get() }
|
||||||
|
|
||||||
|
// Start makes the writer and starts it as soon as the own name is known: in this call when it already is, else on a goroutine.
|
||||||
|
func (f *lateFindings) Start(ctx context.Context, say func(string)) {
|
||||||
|
begin := func(module string) {
|
||||||
|
w, err := f.make(module)
|
||||||
|
if err != nil { // a writer that cannot be made is a fault of its configuration: loud
|
||||||
|
say("the node-engine cannot make its finding writer: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := w.Start(ctx); err != nil {
|
||||||
|
say("the node-engine's findings are kept in its state and not said as events until it is started again: " + err.Error())
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.w = w
|
||||||
|
f.mu.Unlock()
|
||||||
|
}
|
||||||
|
if name := f.own.Get(); name != "" {
|
||||||
|
begin(name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if name, err := f.own.Wait(ctx); err == nil {
|
||||||
|
begin(name)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
var errOwnNameUnknown = errors.New("the node-engine's own module is not yet named by its applied record or its first composing, so its finding writer is not made")
|
||||||
|
|
||||||
|
func (f *lateFindings) writer() *findingsOfTheNode {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.w
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Write(ctx context.Context, fi condseat.Finding) error {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Write(ctx, fi)
|
||||||
|
}
|
||||||
|
return errOwnNameUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Clear(ctx context.Context, scope, id, kind, why string) error {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Clear(ctx, scope, id, kind, why)
|
||||||
|
}
|
||||||
|
return errOwnNameUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *lateFindings) Held() []condseat.Finding {
|
||||||
|
if w := f.writer(); w != nil {
|
||||||
|
return w.Held()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// viewWaitsFile is the file the manifest declares with the content ${config:view-waits-minutes} (design 47 G §633), read at
|
||||||
|
// the engine's start as recoveryWaitFile is.
|
||||||
|
var viewWaitsFile = "/var/lib/mesh-host/view-waits-minutes"
|
||||||
|
|
||||||
|
// viewWaitsBound is how long the reader of the seat list may wait before the engine says it: the file's minutes, the design's
|
||||||
|
// five (seatwait.Bound) when the file is missing, empty or not a positive number.
|
||||||
|
func viewWaitsBound() time.Duration {
|
||||||
|
raw, err := os.ReadFile(viewWaitsFile)
|
||||||
|
if err != nil {
|
||||||
|
return seatwait.Bound
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return seatwait.Bound
|
||||||
|
}
|
||||||
|
return time.Duration(n) * time.Minute
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ADR 0636 §1: when the node's reader of the seat list could not start, the composer is gated by a reader that never
|
||||||
|
// allows, where a missing reader would let the node act on everything; the shadow, which applies nothing, stays ungated.
|
||||||
|
func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) {
|
||||||
|
failed := composerEnv(nil, "n1", func(string) {}, composerHooks{seatListFailed: true})
|
||||||
|
if failed.SeatList == nil || failed.SeatList.MayAct() || failed.SeatList.MayAct("node-engine") {
|
||||||
|
t.Fatalf("the composer of a node whose reader did not start is gated by %v, want a gate that allows nothing", failed.SeatList)
|
||||||
|
}
|
||||||
|
if _, ok := failed.SeatList.(selfcompose.Unread); !ok {
|
||||||
|
t.Errorf("gate is %T", failed.SeatList)
|
||||||
|
}
|
||||||
|
if got := composerEnv(nil, "n1", func(string) {}, composerHooks{}).SeatList; got != nil {
|
||||||
|
t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// settledEvent is where the status view says a module has settled on a node: its have reached its want, running and healthy
|
||||||
|
// (design 47; the view's `settled`, said once for a module on a node at one composition). The engine hears it for every module
|
||||||
|
// and tells its agent, which tries again, once per settled, a failed apply that waits on that module as the holder of the store
|
||||||
|
// its image comes from (design 51 *distribution's first delivery*, issue 1966 ENG-11). The engine's manifest consumes it (`mesh-delivery-view.settled`),
|
||||||
|
// and that line alone grants the engine's account the subject (TestTheSettledEventIsTheManifestsConsume); a refused subscription is said, and made again when the next link opens.
|
||||||
|
const settledEvent = "mesh.mod.mesh-delivery-view.event.settled"
|
||||||
|
|
||||||
|
// settledBody is what of the event the engine reads: the module that settled, on whichever node.
|
||||||
|
type settledBody struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledTold is who is told a holder has settled.
|
||||||
|
type settledTold interface{ Settled(holder string) }
|
||||||
|
|
||||||
|
// followSettled hears the status view's `settled` on every link, never on a timer, until ctx ends. now names the agent running
|
||||||
|
// now (nil while the node agent begins again): a settled heard while there is none is not lost for it, since a new agent applies
|
||||||
|
// again every have that failed at its start.
|
||||||
|
func followSettled(ctx context.Context, session *link.Session, now func() settledTold, say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
events, stop, herr := session.Hear(settledEvent)
|
||||||
|
if herr != nil {
|
||||||
|
if line := fmt.Sprintf("a store settling is not heard yet (%s: %v): a failed pull waits for the next link, or for compose", settledEvent, herr); line != said {
|
||||||
|
said = line
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
said = ""
|
||||||
|
listen(linked, events, now, say)
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
<-linked.Done()
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// listen tells the agent each settled it hears, until the link ends.
|
||||||
|
func listen(linked context.Context, events <-chan []byte, now func() settledTold, say link.Announce) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
return
|
||||||
|
case raw, ok := <-events:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var b settledBody
|
||||||
|
if json.Unmarshal(raw, &b) != nil || b.Module == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if told := now(); told != nil {
|
||||||
|
told.Settled(b.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The subject the engine hears for a store settling is the event its manifest consumes: the grants part grants an account
|
||||||
|
// what its manifest consumes and nothing named in code (novox/hq ADR 0625 §4).
|
||||||
|
func TestTheSettledEventIsTheManifestsConsume(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
emitter, event, _ := strings.Cut(strings.TrimPrefix(settledEvent, "mesh.mod."), ".event.")
|
||||||
|
if !slices.Contains(m.Consumes, emitter+"."+event) {
|
||||||
|
t.Errorf("the engine hears %s, its manifest consumes %v", settledEvent, m.Consumes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The manifest claims the seat node-engine, says its findings on its own stream and hears the keeper's ask and the
|
||||||
|
// node-engine seat's running (design 47 G §364, §387, §199).
|
||||||
|
func TestTheManifestClaimsItsSeatAndSaysItsFindings(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
Claims []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
} `json:"claims"`
|
||||||
|
Streams []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Events []string `json:"events"`
|
||||||
|
} `json:"streams"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(m.Claims) != 1 || m.Claims[0].Name != "node-engine" || m.Claims[0].Scope != "node" {
|
||||||
|
t.Errorf("claims = %+v, want the node seat node-engine", m.Claims)
|
||||||
|
}
|
||||||
|
if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-host-findings" ||
|
||||||
|
!slices.Equal(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
|
||||||
|
t.Errorf("streams = %+v, want mesh-host-findings with the three finding events", m.Streams)
|
||||||
|
}
|
||||||
|
for _, c := range []string{"mesh-conditions.findings-asked", "node-engine.running"} {
|
||||||
|
if !slices.Contains(m.Consumes, c) {
|
||||||
|
t.Errorf("consumes %v lacks %s", m.Consumes, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.Version != "0.1" {
|
||||||
|
t.Errorf("version = %q, want MAJOR.MINOR 0.1 (§368, §411)", m.Version)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the
|
||||||
|
// record, so the module's kept data and its record survive (design 47 G A-98).
|
||||||
|
func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) {
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
path := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
var s store.State
|
||||||
|
s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"})
|
||||||
|
s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"})
|
||||||
|
if err := store.Save(path, s); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var asked []string
|
||||||
|
opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}}
|
||||||
|
if res := stopComposition(context.Background(), opts, "app"); res.Failed {
|
||||||
|
t.Fatalf("stop failed: %s", res.Detail)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") ||
|
||||||
|
strings.Contains(strings.Join(asked, "\n"), " rm ") {
|
||||||
|
t.Fatalf("the machine was asked %v", asked)
|
||||||
|
}
|
||||||
|
if after, err := store.Load(path); err != nil || len(after.Resources) != 2 {
|
||||||
|
t.Fatalf("the record changed: %v, %v", after.Resources, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stopBound is how long a stopping engine waits for the apply in flight to end and save what it did: under the service
|
||||||
|
// manager's own stop bound (90 s by default, then every process of the unit is killed), so the wait ends with a line in
|
||||||
|
// the journal rather than with a kill.
|
||||||
|
const stopBound = 60 * time.Second
|
||||||
|
|
||||||
|
// stopBetweenApplies is how this engine ends, whichever way (a stop of the service, standing aside for a successor, a
|
||||||
|
// move to another bus): it waits, up to bound, for the apply in flight to end, then lets no save start and waits for
|
||||||
|
// the one writing (novox/hq issue 1585). An apply runs detached from the engine's own stop (applyContext), so without
|
||||||
|
// this the process ended under it: what it unpacked was never recorded, and a save cut between its write and its
|
||||||
|
// rename left a temporary file and the state before it. A stopping engine never gives back what it takes (the process
|
||||||
|
// ends); the release it returns is for a test, and gives back the apply only when this took it.
|
||||||
|
func stopBetweenApplies(say func(string), bound time.Duration) (release func()) {
|
||||||
|
took := true
|
||||||
|
if !applying.TryLock() {
|
||||||
|
say(fmt.Sprintf("stopping: waiting up to %s for the apply in flight to end and save what it did", bound))
|
||||||
|
got := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
applying.Lock()
|
||||||
|
close(got)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-got:
|
||||||
|
say("stopping: the apply in flight ended and its state is saved")
|
||||||
|
case <-time.After(bound):
|
||||||
|
took = false
|
||||||
|
say(fmt.Sprintf("stopping while an apply still runs after %s: what it did since its last save is not "+
|
||||||
|
"recorded, and the next start applies its module again", bound))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saves := store.Quiesce()
|
||||||
|
return func() {
|
||||||
|
saves()
|
||||||
|
if took {
|
||||||
|
applying.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeLeftoverSaves removes, at the engine's start, the temporary files of saves of the node's state that never
|
||||||
|
// reached their rename, and says how many (issue 1585: about 260 on novox, ~80 MB).
|
||||||
|
func removeLeftoverSaves(statePath string, say func(string)) {
|
||||||
|
removed, err := store.RemoveLeftovers(statePath)
|
||||||
|
if len(removed) > 0 {
|
||||||
|
shown := removed
|
||||||
|
if len(shown) > 5 {
|
||||||
|
shown = append(append([]string(nil), shown[:5]...), "…")
|
||||||
|
}
|
||||||
|
say(fmt.Sprintf("removed %d leftover save(s) of this node's state, cut off before their rename by an earlier "+
|
||||||
|
"stop: %s", len(removed), strings.Join(shown, ", ")))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say(err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// exitBetweenApplies ends the process cleanly once no apply runs and no save writes: an exit that is not runLink's
|
||||||
|
// return (a move to another bus). A variable, so a test sees it called rather than the test process ending.
|
||||||
|
var exitBetweenApplies = func(say func(string)) {
|
||||||
|
_ = stopBetweenApplies(say, stopBound)
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d; issue 1126): the
|
||||||
|
// vault's key alone, read from the vault's state `signers`, which only the vault may write: the bus grants are its
|
||||||
|
// anchor, as the wants' and the assignment's. The controller's key, which the node was enrolled with (its membership's
|
||||||
|
// signer), is trusted never: the controller ended by its unassignment (the operator, 2026-11-02) and its lease is never
|
||||||
|
// handed, so no gate waits on it. What it trusts now the node states in its record (NodeRecord.Trusts).
|
||||||
|
type meshTrust struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
vault ed25519.PublicKey
|
||||||
|
authority string
|
||||||
|
// changed is told when what it trusts changed, so the record is written again.
|
||||||
|
changed func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMeshTrust trusts nothing until the vault's key is read.
|
||||||
|
func newMeshTrust() *meshTrust { return &meshTrust{} }
|
||||||
|
|
||||||
|
// onChange names who is told when what it trusts changed.
|
||||||
|
func (t *meshTrust) onChange(f func()) {
|
||||||
|
t.mu.Lock()
|
||||||
|
t.changed = f
|
||||||
|
t.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signers are the keys a mesh signature is verified against now: the vault's once read, and none before.
|
||||||
|
func (t *meshTrust) Signers() []meshvaultclient.Signer {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
var out []meshvaultclient.Signer
|
||||||
|
if t.vault != nil {
|
||||||
|
out = append(out, meshvaultclient.Signer{Name: meshvaultclient.SignerVault, Key: t.vault})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify is the signer that signed msg as kind, or why none it trusts did.
|
||||||
|
func (t *meshTrust) Verify(kind string, msg, sig []byte) (meshvaultclient.Signer, error) {
|
||||||
|
context, err := meshvaultclient.SignContext(kind)
|
||||||
|
if err != nil {
|
||||||
|
return meshvaultclient.Signer{}, err
|
||||||
|
}
|
||||||
|
return meshvaultclient.VerifyBy(t.Signers(), context, msg, sig)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
|
||||||
|
func (t *meshTrust) VaultAuthority() string {
|
||||||
|
if t == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.authority
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
|
||||||
|
func (t *meshTrust) Trusts() []string {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
authorities := map[string]string{}
|
||||||
|
if a := t.VaultAuthority(); a != "" {
|
||||||
|
authorities[meshvaultclient.SignerVault] = a
|
||||||
|
}
|
||||||
|
return meshvaultclient.Trusts(t.Signers(), authorities)
|
||||||
|
}
|
||||||
|
|
||||||
|
// change applies f under the lock and tells who follows when Trusts differs after it.
|
||||||
|
func (t *meshTrust) change(f func()) {
|
||||||
|
before := strings.Join(t.Trusts(), "\n")
|
||||||
|
t.mu.Lock()
|
||||||
|
f()
|
||||||
|
changed := t.changed
|
||||||
|
t.mu.Unlock()
|
||||||
|
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
|
||||||
|
changed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
|
||||||
|
// check; a deleted key is no longer trusted.
|
||||||
|
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
|
||||||
|
switch key {
|
||||||
|
case meshvault.SignersByKeyKey():
|
||||||
|
var pub ed25519.PublicKey
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
k, err := meshvaultclient.DecodeSignerKey(value)
|
||||||
|
if err == nil {
|
||||||
|
pub, err = meshvaultclient.SignerPublicKey(k)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's signing key in " + buckets.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.change(func() { t.vault = pub })
|
||||||
|
case meshvault.SignersByAuthorityKey():
|
||||||
|
pem := ""
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
a, err := meshvaultclient.DecodeSignerAuthority(value)
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's authority in " + buckets.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pem = a.Certificate
|
||||||
|
}
|
||||||
|
t.change(func() { t.authority = pem })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
|
||||||
|
// not serve `sign` yet, and until it does this node is not granted the reads.
|
||||||
|
var trustFollowAgain = 2 * time.Minute
|
||||||
|
|
||||||
|
// followTrust follows the vault's `signers` on every link, as watches (never a poll), for as
|
||||||
|
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
|
||||||
|
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
sayOnce := func(line string) {
|
||||||
|
if line != said {
|
||||||
|
said = line
|
||||||
|
if line != "" {
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
watching, stop := context.WithCancel(linked)
|
||||||
|
signers, serr := session.KVWatch(watching, buckets.Signers, ">")
|
||||||
|
if serr != nil {
|
||||||
|
stop()
|
||||||
|
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v): "+
|
||||||
|
"it trusts what it last read, and looks again in %s", serr, trustFollowAgain))
|
||||||
|
} else {
|
||||||
|
sayOnce("")
|
||||||
|
for u := range signers {
|
||||||
|
if !u.Caught {
|
||||||
|
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
case <-time.After(trustFollowAgain):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedAllHave says whether every module the seat mesh-module-assigner places on node has a have of its own, read from the
|
||||||
|
// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose assignment or haves
|
||||||
|
// cannot be read, keeps reading it.
|
||||||
|
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
|
||||||
|
key, err := meshmoduleassignerclient.NodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
// Its node key names every module placed (research 072 row 37).
|
||||||
|
values := map[string][]byte{}
|
||||||
|
if h, err := session.KVHistory(ctx, buckets.Assignments, key); err == nil && len(h) > 0 && !h[len(h)-1].Deleted {
|
||||||
|
values[key] = h[len(h)-1].Value
|
||||||
|
}
|
||||||
|
p, err := placedModules(node, values)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
return allHave(p, func(module string) (nodeengine.HaveByNodeModuleValue, bool, error) {
|
||||||
|
return session.ReadHaveErr(ctx, module)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedModules is the modules node's node key names (an assignment of their names alone); an error when it does not
|
||||||
|
// read.
|
||||||
|
func placedModules(node string, values map[string][]byte) (meshmoduleassigner.Assignments, error) {
|
||||||
|
key, _ := meshmoduleassignerclient.NodeKey(node)
|
||||||
|
raw, ok := values[key]
|
||||||
|
if !ok {
|
||||||
|
return meshmoduleassigner.Assignments{}, errors.New("its assignment is not read")
|
||||||
|
}
|
||||||
|
n, err := meshmoduleassignerclient.DecodeNode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return meshmoduleassigner.Assignments{}, err
|
||||||
|
}
|
||||||
|
p := meshmoduleassigner.Assignments{Node: node}
|
||||||
|
for _, r := range n.Modules {
|
||||||
|
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: r.Module})
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// allHave says whether has finds, for every module p places, a have that names a fingerprint of the new form, and why
|
||||||
|
// not. A module whose have names only a build of the old form runs inside the node's own composition, and the kept
|
||||||
|
// declaration is the one record of what runs of it: retired then, the node's own composed it as nothing and refused to
|
||||||
|
// remove what runs (issue 774, 2026-10-30 19:43: systemd-resolved on shanks, nats and restic on novox, each node's own
|
||||||
|
// apply refused from then on).
|
||||||
|
func allHave(p meshmoduleassigner.Assignments, has func(module string) (nodeengine.HaveByNodeModuleValue, bool, error)) (bool, string) {
|
||||||
|
if len(p.Modules) == 0 {
|
||||||
|
return false, "the seat mesh-module-assigner places no module here"
|
||||||
|
}
|
||||||
|
for _, m := range p.Modules {
|
||||||
|
h, ok, err := has(m.Module)
|
||||||
|
if err != nil {
|
||||||
|
return false, "the have of " + m.Module + " is not read: " + err.Error()
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return false, m.Module + " has no have yet"
|
||||||
|
}
|
||||||
|
// A module the mesh provides (mesh-wireguard) never has a fingerprint: the node's own composes it from the manifest
|
||||||
|
// mesh-sdk provides (overlay.ProvidedManifest), never from the kept declaration, so it keeps no read.
|
||||||
|
if h.Phase != nodeengineclient.PhaseRemoved && !strings.HasPrefix(h.Fp, meshcatalogclient.SourcePrefix2) && !overlay.ProvidedByMesh(m.Module) {
|
||||||
|
return false, m.Module + " runs at a build of the old form (" + h.Build + "), which only the kept declaration records"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
|
||||||
|
}
|
||||||
|
|
||||||
|
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
|
||||||
|
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
|
||||||
|
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
|
||||||
|
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
|
||||||
|
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
|
||||||
|
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key, err := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||||
|
return fmt.Errorf("this node's serving key is not an ed25519 key")
|
||||||
|
}
|
||||||
|
raw, err := ask(ctx, meshvault.VerbSign, map[string]any{"kind": meshvaultclient.KindNodeCertificate, "node": node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var a meshvault.SignAnswer
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return fmt.Errorf("the vault's answer is not a signing: %w", err)
|
||||||
|
}
|
||||||
|
trusted := trust.VaultAuthority()
|
||||||
|
if trusted == "" {
|
||||||
|
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
|
||||||
|
}
|
||||||
|
if err := meshvaultclient.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
|
||||||
|
ed25519.PublicKey(key), time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
path := identity.CertificatePath(state)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if written != nil {
|
||||||
|
written()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/units"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The machine's failed units in the engine's statement (novox/hq issue 315): a module's is among the
|
||||||
|
// resources, unhealthy, as that module's unit, naming it; the machine's own beside them; and an engine
|
||||||
|
// not reading them says nothing of them.
|
||||||
|
func TestTheStatementCarriesTheFailedUnits(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||||
|
us := &units.Statement{At: at, State: units.Degraded, Failed: []units.Failed{
|
||||||
|
{Unit: "openrazer-daemon.service", Scope: units.ScopeUser, Load: "loaded", Result: "exit-code",
|
||||||
|
User: "operator", Module: "openrazer", Resource: "openrazer.daemon", Via: units.ViaPackage, Since: at, Streak: 2},
|
||||||
|
{Unit: "storage-media.mount", Scope: units.ScopeSystem, Load: "loaded", Result: "timeout", Since: at, Streak: 2},
|
||||||
|
}}
|
||||||
|
h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), us)
|
||||||
|
if len(h.Resources) != 1 {
|
||||||
|
t.Fatalf("one module's unit among the resources: %+v", h.Resources)
|
||||||
|
}
|
||||||
|
r := h.Resources[0]
|
||||||
|
if r.Module != "openrazer" || r.Resource != "openrazer.daemon" || r.Kind != link.KindUnit ||
|
||||||
|
r.Target != "openrazer-daemon.service" || r.State != link.StateUnhealthy ||
|
||||||
|
!strings.Contains(r.Reason, "account's own service manager") || !strings.Contains(r.Reason, "exit-code") {
|
||||||
|
t.Fatalf("the module's failed unit: %+v", r)
|
||||||
|
}
|
||||||
|
// Whose manager it fails in, so the controller can tell it from a fault (novox/hq ADR 0254).
|
||||||
|
if r.Account != "operator" {
|
||||||
|
t.Fatalf("the unit in the account's own manager does not name the account: %q", r.Account)
|
||||||
|
}
|
||||||
|
if h.Units == nil || h.Units.State != units.Degraded || len(h.Units.Failed) != 1 ||
|
||||||
|
h.Units.Failed[0].Unit != "storage-media.mount" || h.Units.Failed[0].Result != "timeout" {
|
||||||
|
t.Fatalf("the machine's own: %+v", h.Units)
|
||||||
|
}
|
||||||
|
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), nil); h.Units != nil {
|
||||||
|
t.Fatal("an engine not reading units said them")
|
||||||
|
}
|
||||||
|
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), &units.Statement{}); h.Units != nil {
|
||||||
|
t.Fatal("a judge not yet given a declaration said units")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every resource that runs in an account's own service manager names the account, and so does the account
|
||||||
|
// a module put in a group (novox/hq ADR 0254): the controller reads a unit that cannot run before a new login
|
||||||
|
// by the two together. What the machine's own manager runs names none.
|
||||||
|
func TestTheStatementNamesTheAccountsManager(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||||
|
st := liveness.Statement{At: at, Resources: []liveness.State{
|
||||||
|
{Resource: liveness.Resource{Module: "openrazer", ID: "openrazer.daemon", Kind: "service",
|
||||||
|
Target: "openrazer-daemon.service", Scope: declaration.ScopeUser, User: "operator"}, State: liveness.Unhealthy,
|
||||||
|
Reason: "down", Since: at},
|
||||||
|
{Resource: liveness.Resource{Module: "docker", ID: "docker.engine", Kind: "service", Target: "docker.service",
|
||||||
|
Scope: declaration.ScopeSystem}, State: liveness.Healthy, Since: at},
|
||||||
|
}}
|
||||||
|
as := &accounts.Statement{At: at, Accounts: []accounts.Verdict{{Account: accounts.Account{Module: "openrazer",
|
||||||
|
ID: "openrazer.operator", Name: "operator", Groups: []string{"openrazer"}}, State: accounts.Unhealthy,
|
||||||
|
Reason: accounts.ReasonRelogin + ": operator is in the group openrazer", Since: at}}}
|
||||||
|
h := withAccounts(healthAsReported(st, nil), as)
|
||||||
|
got := map[string]string{}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
got[r.Resource] = r.Account
|
||||||
|
}
|
||||||
|
want := map[string]string{"openrazer.daemon": "operator", "docker.engine": "", "openrazer.operator": "operator"}
|
||||||
|
for id, account := range want {
|
||||||
|
if a, said := got[id]; !said || a != account {
|
||||||
|
t.Errorf("%s names the account %q (said %v); want %q", id, a, said, account)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An account declared never to become root (novox/hq ADR 0266) is said with root never, under the contract
|
||||||
|
// that tells the controller this engine judges it; an account judged for its groups alone is not.
|
||||||
|
func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||||
|
as := &accounts.Statement{At: at, Accounts: []accounts.Verdict{
|
||||||
|
{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
|
||||||
|
State: accounts.Unhealthy, Reason: accounts.ReasonRoot + ": in the group docker, which grants root", Since: at},
|
||||||
|
{Account: accounts.Account{Module: "openrazer", ID: "openrazer.account", Name: "operator",
|
||||||
|
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
|
||||||
|
}}
|
||||||
|
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||||
|
// At RootContract or later: an engine that states its sealing key to the vault says NodeKeyContract (ADR 0372, issue 786), one above
|
||||||
|
// VaultBoxContract (it opens the vault's box for the core's credentials, ADR 0349 §16, issue 755), one above HandedContract (design 57 P7), one above ProvisionerContract (novox/hq ADR
|
||||||
|
// 0349 §4), one above ADR 0349 §5's OwnershipContract.
|
||||||
|
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.NodeKeyContract ||
|
||||||
|
link.NodeKeyContract != link.VaultBoxContract+1 || link.VaultBoxContract != link.HandedContract+1 ||
|
||||||
|
link.HandedContract != link.ProvisionerContract+1 || link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 ||
|
||||||
|
link.BusyContract != link.OwnGroupContract+1 {
|
||||||
|
t.Fatalf("contract %d", h.Contract)
|
||||||
|
}
|
||||||
|
got := map[string]string{}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
got[r.Resource] = r.Root
|
||||||
|
}
|
||||||
|
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
||||||
|
t.Fatalf("root said: %v", got)
|
||||||
|
}
|
||||||
|
// 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old
|
||||||
|
// with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why.
|
||||||
|
t.Run("this machine's root-free", func(t *testing.T) {
|
||||||
|
agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
|
||||||
|
State: accounts.Healthy, Since: at}
|
||||||
|
free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
st *accounts.Statement
|
||||||
|
after time.Duration
|
||||||
|
served bool
|
||||||
|
free bool
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"},
|
||||||
|
{name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"},
|
||||||
|
{name: "no verdict", st: nil, why: "judged no account yet"},
|
||||||
|
{name: "a way to root", st: as, why: "in the group docker, which grants root"},
|
||||||
|
{name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"},
|
||||||
|
{name: "execute served", st: free, served: true, why: "the login shell runs any command"},
|
||||||
|
} {
|
||||||
|
v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here")
|
||||||
|
if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" {
|
||||||
|
t.Errorf("%s: %+v", row.name, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// execute is served while the holder's setting file says anything but withhold, or the bus hears it.
|
||||||
|
dir := t.TempDir()
|
||||||
|
statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf")
|
||||||
|
if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
quiet := func() (bool, error) { return false, nil }
|
||||||
|
for _, row := range []struct {
|
||||||
|
name, setting string
|
||||||
|
heard func() (bool, error)
|
||||||
|
served bool
|
||||||
|
}{
|
||||||
|
{name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet},
|
||||||
|
{name: "served", setting: "execute=serve\n", heard: quiet, served: true},
|
||||||
|
{name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true},
|
||||||
|
{name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true},
|
||||||
|
{name: "no setting file", heard: quiet, served: true},
|
||||||
|
} {
|
||||||
|
_ = os.Remove(conf)
|
||||||
|
if row.setting != "" {
|
||||||
|
if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served, why := executeServedHere(statePath, row.heard); served != row.served {
|
||||||
|
t.Errorf("%s: served %v (%s)", row.name, served, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
||||||
|
// fields (novox/hq issue 361); a mount point, never a path the walk visited or found.
|
||||||
|
func TestTheStatementSaysHowFarTheSetuidSearchGot(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 10, 1, 40, 0, 0, time.UTC)
|
||||||
|
sp := &accounts.SearchProgress{Started: at, Filesystems: 3, Done: 1, Walking: "/home", Visited: 42, Found: 2,
|
||||||
|
Kept: at, Finished: []accounts.SearchedFilesystem{{Mount: "/", Took: 90 * time.Second}}}
|
||||||
|
as := &accounts.Statement{At: at, Accounts: []accounts.Verdict{{Account: accounts.Account{Module: "claude-code",
|
||||||
|
ID: "claude-code.agent", Name: "agent", Root: true}, State: accounts.Unknown,
|
||||||
|
Reason: accounts.ReasonPending + ": the search for setuid programs, started 2026-10-10 01:27, has not finished yet",
|
||||||
|
Since: at, Search: sp}}}
|
||||||
|
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||||
|
r := h.Resources[0]
|
||||||
|
if !strings.HasPrefix(r.Reason, accounts.ReasonPending) || !strings.Contains(r.Reason, "1 of 3 filesystems walked, walking /home, 42 entries visited") {
|
||||||
|
t.Fatalf("the reason: %s", r.Reason)
|
||||||
|
}
|
||||||
|
s := r.Search
|
||||||
|
if s == nil || s.Filesystems != 3 || s.Done != 1 || s.Walking != "/home" || s.Visited != 42 || s.Found != 2 ||
|
||||||
|
s.Kept == nil || len(s.Finished) != 1 || s.Finished[0].Mount != "/" || *s.Finished[0].TookSeconds != 90 {
|
||||||
|
t.Fatalf("the fields: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,64 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The last apply's report, kept beside the state until the mesh takes it (novox/hq issue 264).
|
|
||||||
|
|
||||||
// What a host that died between applying and reporting kept is read back by the next one exactly —
|
|
||||||
// the digest and the outcome, never a new account of the machine.
|
|
||||||
func TestTheKeptReportSurvivesTheHost(t *testing.T) {
|
|
||||||
state := filepath.Join(t.TempDir(), "state.json")
|
|
||||||
made := link.Report{Declared: "sha256:d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"},
|
|
||||||
Host: "v2"}
|
|
||||||
if err := unsaidBeside(state).Keep(made); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
kept, ok, err := unsaidBeside(state).Pending()
|
|
||||||
if err != nil || !ok {
|
|
||||||
t.Fatalf("nothing kept: %v", err)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(kept, made) {
|
|
||||||
t.Fatalf("kept %+v, made %+v", kept, made)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Taken about another declaration, it stays; taken about this one, it goes.
|
|
||||||
if err := unsaidBeside(state).Said("sha256:other"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, ok, _ := unsaidBeside(state).Pending(); !ok {
|
|
||||||
t.Fatal("a report about another declaration cleared this one")
|
|
||||||
}
|
|
||||||
if err := unsaidBeside(state).Said("sha256:d1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(store.UnsaidPath(state)); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("the report was taken and is still kept: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A node that never applied anything has nothing to say again; and an apply refused before its
|
|
||||||
// declaration was read clears an older report, which would otherwise be re-said after it.
|
|
||||||
func TestNothingKeptWhenNothingWasApplied(t *testing.T) {
|
|
||||||
state := filepath.Join(t.TempDir(), "state.json")
|
|
||||||
if _, ok, err := unsaidBeside(state).Pending(); ok || err != nil {
|
|
||||||
t.Fatalf("a node that applied nothing has a report to say: %v %v", ok, err)
|
|
||||||
}
|
|
||||||
if err := unsaidBeside(state).Keep(link.Report{Declared: "sha256:d1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := unsaidBeside(state).Keep(link.Report{Refused: "forged"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, ok, _ := unsaidBeside(state).Pending(); ok {
|
|
||||||
t.Fatal("an older report is kept after a later declaration was refused")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
"github.com/novox/mesh-host/internal/userlist"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the grants part's user list alone into the file
|
||||||
|
// the bus module's composition declares, under the apply's two locks as a filled file is written (filling.go), and says what
|
||||||
|
// it wrote in the bus module's have.
|
||||||
|
|
||||||
|
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
||||||
|
// agent trusts, the write alone, and the have.
|
||||||
|
func userListFollower(opts options, session *link.Session, running func() []engcomp.Composed,
|
||||||
|
say link.Announce, wrote func()) *userlist.Follower {
|
||||||
|
return userlist.New(userlist.Env{
|
||||||
|
Node: session.Membership.Node,
|
||||||
|
Running: running,
|
||||||
|
OnDisk: userListOnDisk,
|
||||||
|
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
||||||
|
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
||||||
|
if changed && wrote != nil {
|
||||||
|
wrote()
|
||||||
|
}
|
||||||
|
return digest, changed, err
|
||||||
|
},
|
||||||
|
Report: func(ctx context.Context, module string, w userlist.Written) {
|
||||||
|
session.NoteUserList(ctx, module, w)
|
||||||
|
},
|
||||||
|
// From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and
|
||||||
|
// opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it.
|
||||||
|
Grants: bucketWatch{session, grantsstate.GrantsUserListBucket},
|
||||||
|
Fetch: func(ctx context.Context, name string) ([]byte, error) {
|
||||||
|
return session.GrantsObject(ctx, name, grantsListLimit)
|
||||||
|
},
|
||||||
|
Open: func(b seal.Box) ([]byte, error) {
|
||||||
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return key.OpenVaultBox(b)
|
||||||
|
},
|
||||||
|
Key: func() string {
|
||||||
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
pem, err := key.PublicPEM()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return pem
|
||||||
|
},
|
||||||
|
// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
|
||||||
|
// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
|
||||||
|
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
|
||||||
|
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
|
||||||
|
Backoff: link.Backoff,
|
||||||
|
Say: announceOr(say),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes.
|
||||||
|
const grantsListLimit = 16 << 20
|
||||||
|
|
||||||
|
// userListFile is a user-list resource as the file it declares.
|
||||||
|
func userListFile(raw []byte) (*declaration.File, error) {
|
||||||
|
r, err := declaration.ParseResource(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the user list is not a resource: %w", err)
|
||||||
|
}
|
||||||
|
f, ok := r.(*declaration.File)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("the user list is a %s, not a file", r.Kind())
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// userListOnDisk is what the user list's file holds on this machine now; nil when there is none.
|
||||||
|
func userListOnDisk(raw json.RawMessage) ([]byte, error) {
|
||||||
|
f, err := userListFile(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(f.Path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return body, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeUserListAlone writes one user-list resource alone (apply.WriteAlone) with body as its content, the node's state
|
||||||
|
// read and saved under the apply's locks, and answers the digest of the file on the machine after.
|
||||||
|
func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []byte, say link.Announce) (string, bool, error) {
|
||||||
|
declared, err := userListFile(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
file := *declared
|
||||||
|
file.Content, file.Digest, file.Size = string(body), "", 0
|
||||||
|
|
||||||
|
applying.Lock()
|
||||||
|
defer applying.Unlock()
|
||||||
|
unlock, err := store.Lock(opts.state, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
built, err := system.For(builtFor)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
out, updated, writeErr := apply.WriteAlone(ctx, built, &file, known, apply.ExecRunner, announceOr(say))
|
||||||
|
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||||
|
if err := store.Save(opts.state, updated); err != nil {
|
||||||
|
writeErr = errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+err.Error()))
|
||||||
|
}
|
||||||
|
digest := ""
|
||||||
|
if body, err := os.ReadFile(file.Path); err == nil {
|
||||||
|
digest = userlist.Digest(body)
|
||||||
|
}
|
||||||
|
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
|
||||||
|
func grantsKeyPath(opts options) string {
|
||||||
|
return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
type keptKey struct {
|
||||||
|
Revision uint64 `json:"revision"`
|
||||||
|
Value json.RawMessage `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func keptGrantsKey(opts options) ([]byte, uint64) {
|
||||||
|
raw, err := os.ReadFile(grantsKeyPath(opts))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0
|
||||||
|
}
|
||||||
|
var k keptKey
|
||||||
|
if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
|
||||||
|
return nil, 0
|
||||||
|
}
|
||||||
|
return k.Value, k.Revision
|
||||||
|
}
|
||||||
|
|
||||||
|
func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
|
||||||
|
path := grantsKeyPath(opts)
|
||||||
|
if value == nil {
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
|
||||||
|
tmp := path + ".new"
|
||||||
|
err := os.WriteFile(tmp, raw, 0o600)
|
||||||
|
if err == nil {
|
||||||
|
err = os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// Issue 755's proof on a real bus (novox/hq ADR 0349 §16): a core credential the vault sealed to this machine crosses
|
||||||
|
// the bus inside the declaration, which the controller carries and cannot open, and this engine opens it with its
|
||||||
|
// sealing key and writes the value. A listener on every subject keeps every byte that crossed; the value is never
|
||||||
|
// among them. Run against the throwaway bus the build module starts for the integration level (MESH_TEST_NATS).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
const coreCredential = "the-provisioner-clients-bus-password-2025"
|
||||||
|
|
||||||
|
func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
// This machine's sealing key, where the engine reads it.
|
||||||
|
state := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
key, err := identity.GenerateSealingKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every byte that crosses.
|
||||||
|
tap, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer tap.Close()
|
||||||
|
var mu sync.Mutex
|
||||||
|
var crossed []string
|
||||||
|
if _, err := tap.Subscribe(">", func(m *nats.Msg) {
|
||||||
|
mu.Lock()
|
||||||
|
crossed = append(crossed, m.Subject+" "+string(m.Data))
|
||||||
|
mu.Unlock()
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = tap.Flush()
|
||||||
|
|
||||||
|
// The engine hears the composition on a subject of the test (the node agent reads it from its bucket).
|
||||||
|
const subject = "mesh.test.vaultbox.composition"
|
||||||
|
engine, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer engine.Close()
|
||||||
|
heard, err := engine.SubscribeSync(subject)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = engine.Flush()
|
||||||
|
|
||||||
|
// The vault's answer to the controller: the value in the ADR 0206 box, sealed to this machine's key. The
|
||||||
|
// controller carries it in the declaration as it is.
|
||||||
|
pub, err := key.PublicPEM()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
box, err := seal.Seal(coreCredential, pub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "broker")
|
||||||
|
body, _ := json.Marshal(map[string]any{"declaration": 1, "resources": []map[string]any{
|
||||||
|
{"id": "need-provisioner-broker", "type": "file", "path": path, "vault-sealed": box}}})
|
||||||
|
controller, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer controller.Close()
|
||||||
|
if err := controller.Publish(subject, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = controller.Flush()
|
||||||
|
|
||||||
|
msg, err := heard.NextMsg(5 * time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the declaration did not reach the engine: %v", err)
|
||||||
|
}
|
||||||
|
d, err := declaration.Parse(msg.Data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sys, err := system.For("arch")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := apply.Apply(context.Background(), sys, d, store.State{}, store.OriginCarried,
|
||||||
|
func(context.Context, string, ...string) (string, error) { return "", nil }, nil, sealOpener(state)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
on, err := os.ReadFile(path)
|
||||||
|
if err != nil || string(on) != coreCredential {
|
||||||
|
t.Fatalf("the engine wrote %q, %v", on, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = tap.Flush()
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
mu.Lock()
|
||||||
|
all := strings.Join(crossed, "\n")
|
||||||
|
mu.Unlock()
|
||||||
|
if !strings.Contains(all, subject) || !strings.Contains(all, "vault-sealed") {
|
||||||
|
t.Fatalf("the listener heard nothing of the declaration:\n%s", all)
|
||||||
|
}
|
||||||
|
for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} {
|
||||||
|
if strings.Contains(all, form) {
|
||||||
|
t.Fatal("the core credential crossed the bus in the clear")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The engine opens a value the controller sealed (the NaCl box) as before, and the vault's box, with one key.
|
||||||
|
func TestTheEnginesOpenerOpensBothBoxes(t *testing.T) {
|
||||||
|
state := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
key, _ := identity.GenerateSealingKey()
|
||||||
|
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open := sealOpener(state)
|
||||||
|
nacl, _ := identity.Seal(key.Public, []byte("old"))
|
||||||
|
if got, err := open(nacl); err != nil || string(got) != "old" {
|
||||||
|
t.Errorf("the controller's box opened to %q, %v", got, err)
|
||||||
|
}
|
||||||
|
pub, _ := key.PublicPEM()
|
||||||
|
box, _ := seal.Seal("new", pub)
|
||||||
|
raw, _ := json.Marshal(box)
|
||||||
|
if got, err := open(string(raw)); err != nil || string(got) != "new" {
|
||||||
|
t.Errorf("the vault's box opened to %q, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,305 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func aFileOf(module, id string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
|
||||||
|
"content": "x", "module": module})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// What runs is the agent's compositions and the kept declaration's resources for each module the agent does not hold;
|
||||||
|
// the node's own parts are the kept declaration's until the agent holds the node's composition (ADR 0539 §9). One module
|
||||||
|
// under the agent must not make the rest invisible to the liveness, units, accounts, libraries, filling and the take act.
|
||||||
|
func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
||||||
|
kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}},
|
||||||
|
Libraries: []declaration.Library{{Library: "media"}}}
|
||||||
|
everything := []engcomp.Composed{
|
||||||
|
{Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}},
|
||||||
|
{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's
|
||||||
|
}
|
||||||
|
d, err := whatRuns(everything, false, func() (*declaration.Declaration, error) { return kept, nil })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(d.Resources) != 2 || len(d.Identities) != 1 || len(d.Libraries) != 1 {
|
||||||
|
t.Fatalf("%d resources, %d identities, %d libraries: the kept modules and the node's kept parts are missing",
|
||||||
|
len(d.Resources), len(d.Identities), len(d.Libraries))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the agent holds the node's composition, its parts are the node's.
|
||||||
|
node := engcomp.Composed{Node: "n1", Own: engcomp.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}}
|
||||||
|
d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(d.Identities) != 1 || d.Identities[0].Name != "mesh-web" || len(d.Libraries) != 0 {
|
||||||
|
t.Fatalf("the node's parts: %+v %+v", d.Identities, d.Libraries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration.
|
||||||
|
func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
||||||
|
t.Run("a fill while the agent restarts", aFillWhileTheAgentRestartsIsWrittenAfterwards)
|
||||||
|
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
|
return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil
|
||||||
|
})
|
||||||
|
defer setRunsHere(nil)
|
||||||
|
d, err := declaredNow("/nowhere", nil)
|
||||||
|
if err != nil || d.Provisioner != "from-agent" {
|
||||||
|
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the health loop's judges are handed what runs once a module settles (g14 and shanks, 2026-10-30 16:14):
|
||||||
|
// restic's composition dropped its process restic.measure, which stayed watched and said down on every look.
|
||||||
|
j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), fakeRuntime{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
judging = j
|
||||||
|
defer func() { judging = nil }()
|
||||||
|
process := func(name string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": "restic." + name, "type": "process", "name": "restic-" + name,
|
||||||
|
"source": "https://store.invalid/restic", "digest": "sha256:" + strings.Repeat("a", 64),
|
||||||
|
"run": []string{"./restic"}, "module": "restic"})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
runs []string
|
||||||
|
watch string
|
||||||
|
}{{[]string{"night", "measure"}, "restic.measure restic.night"}, {[]string{"night"}, "restic.night"}} {
|
||||||
|
var raws []json.RawMessage
|
||||||
|
for _, name := range row.runs {
|
||||||
|
raws = append(raws, process(name))
|
||||||
|
}
|
||||||
|
runs, err := whatRuns([]engcomp.Composed{{Node: "n1", Module: "restic", Resources: raws}}, false,
|
||||||
|
func() (*declaration.Declaration, error) { return nil, errors.New("none kept") })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { return runs, nil })
|
||||||
|
judgeWhatRunsNow(filepath.Join(t.TempDir(), "state.json"), nil)
|
||||||
|
st, _ := j.Look(context.Background())
|
||||||
|
var watched []string
|
||||||
|
for _, r := range st.Resources {
|
||||||
|
watched = append(watched, r.ID)
|
||||||
|
}
|
||||||
|
sort.Strings(watched)
|
||||||
|
if got := strings.Join(watched, " "); got != row.watch {
|
||||||
|
t.Errorf("running %v, the judge watches %q, want %q", row.runs, got, row.watch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
|
||||||
|
// file is there and readable and the controller's key is dropped; until then it reads it.
|
||||||
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
|
||||||
|
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
|
||||||
|
t.Fatalf("before the read is retired: %+v, %v", d, err)
|
||||||
|
}
|
||||||
|
placed := meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
|
||||||
|
// Issue 774, 2026-10-30 19:43: a have naming only a build of the old form (systemd-resolved on shanks, folded in the
|
||||||
|
// node's own) keeps the kept declaration read, since it is the one record of what runs of it.
|
||||||
|
newForm := nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning}
|
||||||
|
oldForm := nodeengine.HaveByNodeModuleValue{Build: "build-1791404191702719760", Phase: nodeengineclient.PhaseRunning}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
has map[string]nodeengine.HaveByNodeModuleValue
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"every placed module has a have of the new form", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm, "lemurs": newForm}, true},
|
||||||
|
{"one has none yet", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm}, false},
|
||||||
|
{"one runs at a build of the old form", map[string]nodeengine.HaveByNodeModuleValue{"zsh": newForm, "lemurs": oldForm}, false},
|
||||||
|
} {
|
||||||
|
if got, why := allHave(placed, func(m string) (nodeengine.HaveByNodeModuleValue, bool, error) { h, ok := row.has[m]; return h, ok, nil }); got != row.want {
|
||||||
|
t.Errorf("%s: %v (%s)", row.name, got, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The module the mesh provides never has a fingerprint, and keeps no read (review of issue 774).
|
||||||
|
withWireguard := meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "zsh"}, {Module: overlay.Name}}}
|
||||||
|
if got, why := allHave(withWireguard, func(m string) (nodeengine.HaveByNodeModuleValue, bool, error) {
|
||||||
|
if m == overlay.Name {
|
||||||
|
return oldForm, true, nil
|
||||||
|
}
|
||||||
|
return newForm, true, nil
|
||||||
|
}); !got {
|
||||||
|
t.Errorf("%s without a fingerprint beside zsh at the new form keeps the read: %s", overlay.Name, why)
|
||||||
|
}
|
||||||
|
if got, _ := allHave(meshmoduleassigner.Assignments{Node: "n1"}, nil); got {
|
||||||
|
t.Error("a node with nothing placed retires the read")
|
||||||
|
}
|
||||||
|
keptRetired.Store(true)
|
||||||
|
defer keptRetired.Store(false)
|
||||||
|
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
|
||||||
|
t.Errorf("after the read is retired: %v", err)
|
||||||
|
}
|
||||||
|
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
|
||||||
|
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
||||||
|
// the module's: the declaration joined of both names each file once, so the filling writes it; the node's own resources
|
||||||
|
// no module declares stay. (Live from 2026-10-25 12:43: "2 resources declare X" on every node.)
|
||||||
|
func TestTheNodesOwnCopyOfAModulesResourceIsLeftOut(t *testing.T) {
|
||||||
|
own := engcomp.Composed{Node: "n1", Resources: []json.RawMessage{aFileOf("zsh", "rc"), aFileOf("mesh-wireguard", "conf")},
|
||||||
|
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||||
|
zsh := engcomp.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")},
|
||||||
|
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||||
|
d, err := declarationOf(own, zsh)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
at := map[string]int{}
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
at[r.Target()]++
|
||||||
|
}
|
||||||
|
if at["/tmp/r6/zsh/rc"] != 1 || at["/tmp/r6/mesh-wireguard/conf"] != 1 || len(d.Resources) != 2 {
|
||||||
|
t.Errorf("the joined declaration declares %v, want each path once", at)
|
||||||
|
}
|
||||||
|
if !d.Fills("/tmp/r6/zsh/rc") {
|
||||||
|
t.Errorf("the filled path is no longer named")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same at the node's own apply: what the others declare is theirs.
|
||||||
|
trimmed, dropped := ownWithout(own, []engcomp.Composed{zsh})
|
||||||
|
if len(trimmed.Resources) != 1 || len(dropped) != 1 || dropped[0] != "zsh.rc (zsh)" {
|
||||||
|
t.Errorf("the node's own without zsh's: %d resources, dropped %v", len(trimmed.Resources), dropped)
|
||||||
|
}
|
||||||
|
if len(own.Resources) != 2 {
|
||||||
|
t.Errorf("ownWithout changed its argument")
|
||||||
|
}
|
||||||
|
// A node's own composition alone is all of it.
|
||||||
|
if d, err := declarationOf(own); err != nil || len(d.Resources) != 2 {
|
||||||
|
t.Errorf("the node's own alone: %v (%v)", d, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC18 (unit): the kept declaration's filled paths go to the module whose file is at each path, the node's own to
|
||||||
|
// the node's; and what runs from it, joined with the node's composition, names them filled.
|
||||||
|
func TestTheKeptDeclarationsFilledPathsGoToTheirModules(t *testing.T) {
|
||||||
|
raw := []byte(`{"resources":[
|
||||||
|
{"id":"claude-code.settings","type":"file","module":"claude-code","path":"/etc/cc.json"},
|
||||||
|
{"id":"screen-lock.conf","type":"file","module":"screen-lock","path":"/etc/lock.conf"},
|
||||||
|
{"id":"host.motd","type":"file","path":"/etc/motd"},
|
||||||
|
{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc"}],
|
||||||
|
"filled_files":["/etc/cc.json","/etc/lock.conf","/etc/motd"]}`)
|
||||||
|
by, filled := splitKept(raw)
|
||||||
|
if len(by["claude-code"]) != 1 || len(by[agent.NodeModule]) != 1 {
|
||||||
|
t.Fatalf("resources by module: %v", by)
|
||||||
|
}
|
||||||
|
if strings.Join(filled["claude-code"], ",") != "/etc/cc.json" || strings.Join(filled["screen-lock"], ",") != "/etc/lock.conf" ||
|
||||||
|
strings.Join(filled[agent.NodeModule], ",") != "/etc/motd" || len(filled["zsh"]) != 0 {
|
||||||
|
t.Fatalf("filled by module: %v", filled)
|
||||||
|
}
|
||||||
|
cc := engcomp.Composed{Node: "n1", Module: "claude-code", Resources: by["claude-code"], Filled: filled["claude-code"]}
|
||||||
|
node := engcomp.Composed{Node: "n1"}
|
||||||
|
d, err := whatRuns([]engcomp.Composed{cc, node}, true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||||
|
if err != nil || !d.Fills("/etc/cc.json") {
|
||||||
|
t.Fatalf("what runs does not name claude-code's file filled: %v (%v)", d, err)
|
||||||
|
}
|
||||||
|
if splitBy, _ := splitKept([]byte("not json")); splitBy != nil {
|
||||||
|
t.Errorf("a kept declaration that does not read split into %v", splitBy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC17 (unit, review): a resource of the node's own that shares only its target with a module's (another id: a
|
||||||
|
// unit or a container of the same name) is not the module's, and is neither dropped nor left to be removed.
|
||||||
|
func TestASharedTargetWithAnotherIDIsNotDropped(t *testing.T) {
|
||||||
|
ownFile, _ := json.Marshal(map[string]any{"id": "host.zshrc", "type": "file", "path": "/tmp/r6/zsh/rc", "content": "x"})
|
||||||
|
own := engcomp.Composed{Node: "n1", Resources: []json.RawMessage{ownFile}}
|
||||||
|
zsh := engcomp.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")}}
|
||||||
|
trimmed, dropped := ownWithout(own, []engcomp.Composed{zsh})
|
||||||
|
if len(dropped) != 0 || len(trimmed.Resources) != 1 {
|
||||||
|
t.Errorf("a resource sharing only its target was dropped: %v", dropped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 924: the bus link drops, the agent restarts, and the filling is asked while it holds nothing yet. What runs is
|
||||||
|
// then the last view the node held, never the retired kept declaration; and a fill refused for want of any is owed and
|
||||||
|
// told once the agent holds the node again, so every file is written afterwards.
|
||||||
|
func aFillWhileTheAgentRestartsIsWrittenAfterwards(t *testing.T) {
|
||||||
|
keptRetired.Store(true)
|
||||||
|
defer keptRetired.Store(false)
|
||||||
|
node := engcomp.Composed{Node: "n1", Own: engcomp.NodeOwn{Provisioner: "mesh-node-provisioner"}}
|
||||||
|
files := []engcomp.Composed{{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}, Filled: []string{"/tmp/r6/db/a"}}}
|
||||||
|
view := &runsView{everything: append(files, node), holdsNode: true}
|
||||||
|
|
||||||
|
var placed []string
|
||||||
|
// The agent holds the node, then the link drops and it stops: set(nil) keeps its view.
|
||||||
|
holding := true
|
||||||
|
current := &agentNow{view: func(*agent.Agent) *runsView {
|
||||||
|
if holding {
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
return &runsView{}
|
||||||
|
}}
|
||||||
|
current.set(agent.New(agent.Env{}))
|
||||||
|
e := &filling.Engine{
|
||||||
|
Declared: func() (*declaration.Declaration, error) { return runsFromAgent(current)("/nowhere", nil) },
|
||||||
|
UIDOf: func(string) (uint32, error) { return 4242, nil },
|
||||||
|
Lock: func() (func(), error) { return func() {}, nil },
|
||||||
|
Place: func(_ context.Context, _ *declaration.Declaration, path string, _ []byte) (string, string, error) {
|
||||||
|
placed = append(placed, path)
|
||||||
|
return "updated", "d", nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fill := func() filling.Answer {
|
||||||
|
return e.Write(context.Background(), 4242, filling.Request{Path: "/tmp/r6/db/a", Content: []byte("x")})
|
||||||
|
}
|
||||||
|
socket := filepath.Join(t.TempDir(), "sock")
|
||||||
|
if err := os.WriteFile(socket, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The next agent has read no haves and the previous one's view is gone (a cold start): refused, and owed.
|
||||||
|
cold := &agentNow{}
|
||||||
|
cold.set(agent.New(agent.Env{}))
|
||||||
|
cold.set(nil)
|
||||||
|
e.Declared = func() (*declaration.Declaration, error) { return runsFromAgent(cold)("/nowhere", nil) }
|
||||||
|
if a := fill(); a.Failed == "" && a.Refused == "" {
|
||||||
|
t.Fatalf("a fill with nothing read was accepted: %+v", a)
|
||||||
|
}
|
||||||
|
// The link drops: set(nil) keeps the view the agent held, and the fill during the restart is written.
|
||||||
|
e.Declared = func() (*declaration.Declaration, error) { return runsFromAgent(current)("/nowhere", nil) }
|
||||||
|
current.set(nil)
|
||||||
|
if told, err := e.Knock(socket); err != nil || told == "" {
|
||||||
|
t.Fatalf("the refused fill was not owed and told: %q, %v", told, err)
|
||||||
|
}
|
||||||
|
if a := fill(); a.Failed != "" || a.Refused != "" || len(placed) != 1 {
|
||||||
|
t.Fatalf("the fill during the restart was not written: %+v, placed %v", a, placed)
|
||||||
|
}
|
||||||
|
// The new agent holds the node: its own view answers and the kept one is dropped.
|
||||||
|
next := agent.New(agent.Env{})
|
||||||
|
current.set(next)
|
||||||
|
holding = true
|
||||||
|
if _, err := runsFromAgent(current)("/nowhere", nil); err != nil || current.lastView() != nil {
|
||||||
|
t.Fatalf("the kept view stays after the new agent holds the node: %v, %v", err, current.lastView())
|
||||||
|
}
|
||||||
|
}
|
||||||
+33
-13
@@ -41,8 +41,9 @@ func rolledBackHosts(statePath string) []string {
|
|||||||
return upgrade.RolledBackVersions(verdicts)
|
return upgrade.RolledBackVersions(verdicts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// proving is this host waiting for the mesh to take a report it made under its own build — the
|
// proving is this host waiting to write a have key under its own build — the evidence its launcher's trial waits for
|
||||||
// evidence its launcher's trial waits for — and acting on it once.
|
// (novox/hq ADR 0539 §9, ADR 0227 §8: the component replaced is never the only witness of its successor) — and acting
|
||||||
|
// on it once.
|
||||||
type proving struct {
|
type proving struct {
|
||||||
statePath string
|
statePath string
|
||||||
version string
|
version string
|
||||||
@@ -56,24 +57,44 @@ func unproved(statePath, version string) bool {
|
|||||||
return err != nil || known != version
|
return err != nil || known != version
|
||||||
}
|
}
|
||||||
|
|
||||||
// heard is told every account of the machine the mesh has taken. The first one about a declaration,
|
// haveWritten is told when this build has written a have key, or found the bucket already holding what it would write.
|
||||||
// made by this build, proves it.
|
// The first proves it. **A declaration's report does not**: a node-engine that receives none is not on trial for it, and
|
||||||
func (p *proving) heard(r link.Report) {
|
// one that cannot write its haves is what the launcher keeps rolling back.
|
||||||
if r.Declared == "" || r.Refused != "" || r.Host != p.version {
|
func (p *proving) haveWritten() {
|
||||||
return
|
|
||||||
}
|
|
||||||
p.once.Do(func() {
|
p.once.Do(func() {
|
||||||
retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version)
|
retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err))
|
p.say(fmt.Sprintf("this host wrote its first have under its own build %s, and proving it was not complete: %v", p.version, err))
|
||||||
}
|
}
|
||||||
if len(retired) > 0 {
|
if len(retired) > 0 {
|
||||||
p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", ")))
|
p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", ")))
|
||||||
}
|
}
|
||||||
|
renewFallback(p.version, p.say)
|
||||||
renewLauncher(p.say)
|
renewLauncher(p.say)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renewFallback makes the engine placed by hand — the launcher's fallback, and the `nox-mesh-host` on this
|
||||||
|
// machine's PATH — a copy of this proved build when it is not one (novox/hq issue 356). On the same
|
||||||
|
// evidence known-good is written on, and said when it happens or cannot: a stale command that answers
|
||||||
|
// for a build that is not running would otherwise stay until somebody typed it.
|
||||||
|
func renewFallback(version string, say func(string)) {
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
say("the engine placed by hand could not be renewed: this build does not know its own path: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fallback := upgrade.FallbackPath()
|
||||||
|
renewed, err := upgrade.RenewFallback(self, fallback)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
say(fmt.Sprintf("the engine placed by hand at %s is not this build, and could not be renewed: %v", fallback, err))
|
||||||
|
case renewed:
|
||||||
|
say(fmt.Sprintf("renewed the engine placed by hand at %s to this build, %s: the command on this machine is the "+
|
||||||
|
"engine that runs it", fallback, version))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The launcher, as the service manager runs it.
|
// The launcher, as the service manager runs it.
|
||||||
const (
|
const (
|
||||||
launcherPath = upgrade.DefaultLibexec + "/launch"
|
launcherPath = upgrade.DefaultLibexec + "/launch"
|
||||||
@@ -124,10 +145,9 @@ func launcherReplaced(pid int, proc, path string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
||||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) {
|
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
||||||
host, _ := os.Hostname()
|
|
||||||
w := &witness.Watcher{
|
w := &witness.Watcher{
|
||||||
Root: apply.DaemonRoot(), Node: node, Host: host,
|
Root: apply.DaemonRoot(), Node: node,
|
||||||
Asker: func() link.Asker { return queue.Asker() },
|
Asker: func() link.Asker { return queue.Asker() },
|
||||||
Run: witness.Runner(apply.ExecRunner),
|
Run: witness.Runner(apply.ExecRunner),
|
||||||
Hold: func(f func()) {
|
Hold: func(f func()) {
|
||||||
@@ -137,7 +157,7 @@ func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue
|
|||||||
},
|
},
|
||||||
// Said now, not at the next report: the verdict is on every report from here, and the mesh
|
// Said now, not at the next report: the verdict is on every report from here, and the mesh
|
||||||
// is asked for one at once.
|
// is asked for one at once.
|
||||||
Concluded: func(link.Rollback) { queue.ReportAsked() },
|
Concluded: func(link.Rollback) { queue.FactsAsked(ctx) },
|
||||||
Say: say,
|
Say: say,
|
||||||
}
|
}
|
||||||
w.Watch(ctx)
|
w.Watch(ctx)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -12,31 +13,55 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/upgrade"
|
"github.com/novox/mesh-host/internal/upgrade"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The first account the mesh takes from this build, about a declaration, proves it to the launcher;
|
// The first have this build writes proves it to the launcher; nothing else does — not a report about a declaration, not
|
||||||
// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8).
|
// a facts event, not the haves of another build (novox/hq ADR 0539 §9, ADR 0227 §8). R6-A AC7.
|
||||||
func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
|
func TestOnlyAHaveUnderThisBuildProvesIt(t *testing.T) {
|
||||||
state := filepath.Join(t.TempDir(), "state.json")
|
state := filepath.Join(t.TempDir(), "state.json")
|
||||||
t.Setenv("MESH_HOST_LIBEXEC", t.TempDir())
|
t.Setenv("MESH_HOST_LIBEXEC", t.TempDir())
|
||||||
|
// Never the machine's own engine placed by hand: proving renews it (issue 356).
|
||||||
|
fallback := filepath.Join(t.TempDir(), "nox-mesh-host")
|
||||||
|
if err := os.WriteFile(fallback, []byte("a stale engine placed by hand"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(upgrade.FallbackVar, fallback)
|
||||||
var said []string
|
var said []string
|
||||||
p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }}
|
p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }}
|
||||||
|
|
||||||
for _, r := range []link.Report{
|
// Facts said, a link opened, a declaration of the old kind reported: none of it is a have.
|
||||||
{Host: "2.0", Refused: "no"},
|
if !unproved(state, "2.0") {
|
||||||
{Host: "2.0"},
|
t.Fatal("a build that wrote no have is proved")
|
||||||
{Host: "1.0", Declared: "abc"},
|
|
||||||
} {
|
|
||||||
p.heard(r)
|
|
||||||
if !unproved(state, "2.0") {
|
|
||||||
t.Fatalf("%+v proved this build", r)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
p.heard(link.Report{Host: "2.0", Declared: "abc"})
|
if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" {
|
||||||
|
t.Fatal("the engine placed by hand was renewed before any have")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The session tells the proof when it has a have key saying what it would write.
|
||||||
|
session := &link.Session{HaveWritten: p.haveWritten}
|
||||||
|
_ = session
|
||||||
|
p.haveWritten()
|
||||||
if unproved(state, "2.0") {
|
if unproved(state, "2.0") {
|
||||||
t.Fatalf("an account of a declaration under this build did not prove it (%v)", said)
|
t.Fatalf("the first have under this build did not prove it (%v)", said)
|
||||||
}
|
}
|
||||||
if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" {
|
if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" {
|
||||||
t.Fatalf("known-good is %q", got)
|
t.Fatalf("known-good is %q", got)
|
||||||
}
|
}
|
||||||
|
// **Proved, the engine placed by hand becomes this build** (issue 356): the command on the machine's PATH is
|
||||||
|
// the engine that runs it, and that is said.
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want, _ := os.ReadFile(self)
|
||||||
|
if got, _ := os.ReadFile(fallback); len(want) == 0 || string(got) != string(want) {
|
||||||
|
t.Fatalf("the engine placed by hand was not renewed to this build (%d bytes, want %d)", len(got), len(want))
|
||||||
|
}
|
||||||
|
renewed := false
|
||||||
|
for _, s := range said {
|
||||||
|
renewed = renewed || strings.Contains(s, "renewed the engine placed by hand at "+fallback)
|
||||||
|
}
|
||||||
|
if !renewed {
|
||||||
|
t.Fatalf("the renewal was not said: %q", said)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Every verdict that stands is said, and a reconcile that has one says it unasked.
|
// Every verdict that stands is said, and a reconcile that has one says it unasked.
|
||||||
@@ -51,12 +76,6 @@ func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) {
|
|||||||
got[0].Outcome != link.RolledBack {
|
got[0].Outcome != link.RolledBack {
|
||||||
t.Fatalf("what the report says is %+v", got)
|
t.Fatalf("what the report says is %+v", got)
|
||||||
}
|
}
|
||||||
if !worthSaying(link.Report{Rollbacks: got}) {
|
|
||||||
t.Fatal("a reconcile with a rollback standing does not say it")
|
|
||||||
}
|
|
||||||
if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) {
|
|
||||||
t.Fatal("a rollback reached or ended is not news to the reconcile")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The launcher running this host is known to have been replaced only when it certainly was: a shell
|
// The launcher running this host is known to have been replaced only when it certainly was: a shell
|
||||||
|
|||||||
+26
-33
@@ -2,51 +2,44 @@
|
|||||||
|
|
||||||
## `foundation-first-node.lock`
|
## `foundation-first-node.lock`
|
||||||
|
|
||||||
What a machine must be before a mesh exists — the bootstrap in
|
The one genesis bundle (novox/hq ADR 0331): what a machine must be before a mesh exists — the
|
||||||
[novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole:
|
bootstrap in [novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole, on the bus the
|
||||||
|
mesh runs on:
|
||||||
|
|
||||||
```
|
```
|
||||||
0 a container runtime
|
0 a container runtime, and a filter before anything listens
|
||||||
1 the store runs
|
1 the store runs
|
||||||
2 a database per context `inventory` and `identity`
|
2 a database per context `inventory`, `identity` and `licences`
|
||||||
3 those contexts' schemas mesh-controller migrate
|
3 those contexts' schemas mesh-controller migrate
|
||||||
4 the broker runs with a certificate it generated itself
|
4 the bus's certificate mesh-controller broker certificate
|
||||||
5 the control plane runs mesh-controller serve
|
5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it
|
||||||
|
(ADR 0333), with the controller's account as its first user
|
||||||
|
6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS)
|
||||||
```
|
```
|
||||||
|
|
||||||
**A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is
|
**A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is
|
||||||
exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything
|
exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything
|
||||||
else joins the ordinary way.
|
else joins the ordinary way.
|
||||||
|
|
||||||
This file said it stopped at step 4 for longer than that was true, which is its own small lesson:
|
It is a **template**: `mesh-bootstrap` reads it, makes the store's superuser and the controller's
|
||||||
a comment about what something does not do is a comment nobody updates.
|
account on the bus, and writes them where the template says `bootstrap` (internal/bootstrap,
|
||||||
|
rootsecrets.go). A template whose bus connection it does not recognise is refused at preflight,
|
||||||
|
naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
|
||||||
|
ADR 0331; there is one bundle, and every lab bed names it.
|
||||||
|
|
||||||
Build a host carrying it:
|
**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
|
||||||
|
`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
|
||||||
|
`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
|
||||||
|
port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
|
||||||
|
on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
|
||||||
|
|
||||||
|
Build an installer carrying a builder, and point it at this:
|
||||||
|
|
||||||
```
|
```
|
||||||
make host SYSTEM=arch BUNDLE=examples/foundation-first-node.lock
|
make bootstrap IMAGE=<builder image>
|
||||||
|
mesh-bootstrap --bundle examples/foundation-first-node.lock …
|
||||||
```
|
```
|
||||||
|
|
||||||
**The registry address and digests have to be replaced before this is useful.** They are written
|
**The registry address is the lab's.** The images are written as `192.0.2.250:5000/…@sha256:…`; the
|
||||||
as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here,
|
lab rewrites the third-party ones to upstream references by the same digest, and the installer
|
||||||
one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder
|
replaces the control plane's with the image it built.
|
||||||
to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is
|
|
||||||
part of the target.
|
|
||||||
|
|
||||||
### What was verified, and how
|
|
||||||
|
|
||||||
On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers
|
|
||||||
200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere,
|
|
||||||
the `node` table present with its indexes, the migration recorded, and LavinMQ answering
|
|
||||||
`lavinmqctl status` with AMQP listening on 5672.
|
|
||||||
|
|
||||||
Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time.
|
|
||||||
|
|
||||||
Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both
|
|
||||||
containers because the host creates every container `--restart unless-stopped`
|
|
||||||
(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding
|
|
||||||
nothing to do.
|
|
||||||
|
|
||||||
The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to
|
|
||||||
return, so that it does is a property of the host, and the only way to know it holds is to take
|
|
||||||
the machine away and give it back.
|
|
||||||
|
|||||||
+32
-2
@@ -7,11 +7,12 @@ package examples
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
)
|
)
|
||||||
|
|
||||||
func bundle(t *testing.T) *declaration.Declaration {
|
func bundle(t *testing.T) *declaration.Declaration {
|
||||||
@@ -48,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
|
|||||||
}
|
}
|
||||||
sort.Strings(images)
|
sort.Strings(images)
|
||||||
|
|
||||||
want := []string{"lavinmq", "mesh-controller", "postgres"}
|
want := []string{"mesh-controller", "nats", "pgvector"}
|
||||||
if strings.Join(images, ",") != strings.Join(want, ",") {
|
if strings.Join(images, ",") != strings.Join(want, ",") {
|
||||||
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
|
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
|
||||||
"Adding one is a change to what every first node becomes, and to ADR 0006's "+
|
"Adding one is a change to what every first node becomes, and to ADR 0006's "+
|
||||||
@@ -67,3 +68,32 @@ func TestTheControlPlaneIsCarriedToo(t *testing.T) {
|
|||||||
t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " +
|
t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " +
|
||||||
"foundation and stop")
|
"foundation and stop")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331 §2). Two bundles let every bed
|
||||||
|
// and README name the one that could not raise a mesh, for weeks (issue 146).
|
||||||
|
func TestThereIsOneGenesisBundle(t *testing.T) {
|
||||||
|
bundles, err := filepath.Glob("*.lock")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(bundles) != 1 || bundles[0] != "foundation-first-node.lock" {
|
||||||
|
t.Fatalf("examples/ holds %v; it holds one genesis bundle, foundation-first-node.lock", bundles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The control plane the bundle raises is given the bus, with the account the installer fills and
|
||||||
|
// the certificate it pins (ADR 0331 §4); without it, it crash-loops for ever (issue 146).
|
||||||
|
func TestTheControlPlaneIsGivenTheBus(t *testing.T) {
|
||||||
|
for _, r := range bundle(t).Resources {
|
||||||
|
c, ok := r.(*declaration.Container)
|
||||||
|
if !ok || !strings.Contains(c.Image, "mesh-controller") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(c.Env["MESH_BUS_NATS"], "nats://controller:bootstrap@") || c.Env["MESH_BROKER_CERTIFICATE"] == "" {
|
||||||
|
t.Fatalf("the control plane is given the bus %q and the certificate %q",
|
||||||
|
c.Env["MESH_BUS_NATS"], c.Env["MESH_BROKER_CERTIFICATE"])
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatal("nothing in the bundle starts the control plane")
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,200 +0,0 @@
|
|||||||
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
|
||||||
// built (novox/hq ADR 0106, design 25).
|
|
||||||
//
|
|
||||||
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
|
||||||
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
|
||||||
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
|
||||||
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
|
||||||
// from the controller's first composition onward the file is the controller's to write.
|
|
||||||
//
|
|
||||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
|
||||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
|
||||||
// one directory, of necessity: an include path is resolved relative to the including file's own
|
|
||||||
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
|
||||||
// start.
|
|
||||||
//
|
|
||||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
|
||||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
|
||||||
// authenticates with a password (ADR 0004, design 25 §4).
|
|
||||||
|
|
||||||
{
|
|
||||||
"declaration": 1,
|
|
||||||
"resources": [
|
|
||||||
{
|
|
||||||
"id": "container-runtime",
|
|
||||||
"type": "package",
|
|
||||||
"package": "docker"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "container-runtime-running",
|
|
||||||
"type": "service",
|
|
||||||
"unit": "docker.service",
|
|
||||||
"state": "running",
|
|
||||||
"boot": "enabled"
|
|
||||||
},
|
|
||||||
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
|
||||||
// broker are adopted as modules later and so bind to every interface from the moment they
|
|
||||||
// start; the packet filter that governs who may reach them is a module too, installed a
|
|
||||||
// dozen steps later. Between the two, a control-node facing the network had its store and
|
|
||||||
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
|
||||||
// its own — the same table the filter module will replace wholesale once it can derive one:
|
|
||||||
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
|
||||||
// enrols over, the registry a node pulls from), and let the container runtime's own
|
|
||||||
// networks through the forward chain so containers keep working. A published container port
|
|
||||||
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
|
||||||
// and broker's ports are refused from outside — and a container on this machine dialling a
|
|
||||||
// port this machine publishes reaches it through the runtime's proxy, which IS input, which
|
|
||||||
// is why the bus and the registry are opened in both chains, exactly as the derived ruleset
|
|
||||||
// does.
|
|
||||||
{
|
|
||||||
"id": "base-filter-package",
|
|
||||||
"type": "package",
|
|
||||||
"package": "nftables"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "base-filter",
|
|
||||||
"type": "file",
|
|
||||||
"path": "/etc/nftables.conf",
|
|
||||||
"mode": "0644",
|
|
||||||
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "base-filter-loaded",
|
|
||||||
"type": "service",
|
|
||||||
"unit": "nftables.service",
|
|
||||||
"state": "running",
|
|
||||||
"boot": "enabled",
|
|
||||||
"restart-on": ["base-filter"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "store",
|
|
||||||
"type": "container",
|
|
||||||
"name": "mesh-store",
|
|
||||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
|
||||||
"env": {
|
|
||||||
"POSTGRES_PASSWORD": "bootstrap",
|
|
||||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
|
||||||
},
|
|
||||||
"ports": ["5432:5432"],
|
|
||||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
|
||||||
},
|
|
||||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
|
||||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
|
||||||
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
|
||||||
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
|
||||||
{
|
|
||||||
"id": "store-ready",
|
|
||||||
"type": "action",
|
|
||||||
"in": "mesh-store",
|
|
||||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
|
||||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "inventory-database",
|
|
||||||
"type": "action",
|
|
||||||
"in": "mesh-store",
|
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "identity-database",
|
|
||||||
"type": "action",
|
|
||||||
"in": "mesh-store",
|
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
|
||||||
},
|
|
||||||
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
|
||||||
// created the same way and named the same way — which is the whole of adding a context to the
|
|
||||||
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
|
||||||
{
|
|
||||||
"id": "licences-database",
|
|
||||||
"type": "action",
|
|
||||||
"in": "mesh-store",
|
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "context-schemas",
|
|
||||||
"type": "action",
|
|
||||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
|
||||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
||||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
||||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
||||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
||||||
"migrate"],
|
|
||||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bus-certificate",
|
|
||||||
"type": "action",
|
|
||||||
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
|
||||||
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
|
||||||
// shell and no openssl, and no other image the bundle names has one either. So the program
|
|
||||||
// that needs the certificate writes it — already on this machine, since the schema step ran
|
|
||||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
|
||||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
|
||||||
// to ask an authority of.
|
|
||||||
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
|
||||||
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
|
||||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
|
||||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
||||||
"broker", "certificate", "--into", "/tls"],
|
|
||||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
|
||||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
||||||
"broker", "certificate", "--check", "--into", "/tls"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bus-conf-dir",
|
|
||||||
"type": "directory",
|
|
||||||
"path": "/var/lib/mesh-bus-conf",
|
|
||||||
"mode": "0700"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bus-conf",
|
|
||||||
"type": "file",
|
|
||||||
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
|
||||||
"mode": "0644",
|
|
||||||
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bus-accounts",
|
|
||||||
"type": "file",
|
|
||||||
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
|
||||||
"mode": "0600",
|
|
||||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "broker",
|
|
||||||
"type": "container",
|
|
||||||
"name": "mesh-broker",
|
|
||||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
|
||||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
|
||||||
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
|
||||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "broker-ready",
|
|
||||||
"type": "action",
|
|
||||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
|
||||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "control-plane",
|
|
||||||
"type": "container",
|
|
||||||
"name": "mesh-controller",
|
|
||||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
||||||
"network": "host",
|
|
||||||
"args": ["serve"],
|
|
||||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
|
||||||
"env": {
|
|
||||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
||||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
||||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
||||||
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
|
||||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
|
||||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,36 +1,40 @@
|
|||||||
// foundation-first-node.lock — what a machine must be before a mesh exists.
|
// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle
|
||||||
|
// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25).
|
||||||
//
|
//
|
||||||
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
||||||
// store, a database per context, those contexts' schemas, the broker, and the control plane
|
// filter, a store, a database per context, those contexts' schemas, the bus with its certificate
|
||||||
// running on top of them.
|
// and its first user list, and the control plane running on top of them.
|
||||||
//
|
//
|
||||||
// It stopped before the control plane once, and this comment said so for longer than it was true.
|
// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this
|
||||||
|
// carries the first one — the controller's own account — and the control plane's address names the
|
||||||
|
// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's
|
||||||
|
// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap
|
||||||
|
// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first
|
||||||
|
// node has enrolled the installer places the list the controller composes; from the bus's adoption
|
||||||
|
// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5).
|
||||||
//
|
//
|
||||||
// The broker generates its OWN certificate, in its own image, into a volume it then mounts read
|
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||||
// only. Self-signed, because at this moment there is no mesh to issue one and no public name to
|
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||||
// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint
|
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||||
// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration.
|
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||||
|
// start.
|
||||||
|
//
|
||||||
|
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||||
|
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||||
|
// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through
|
||||||
|
// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback.
|
||||||
//
|
//
|
||||||
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
||||||
// image, and this file is applied on a machine with no mesh to ask about anything. These digests
|
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||||
// belong to the registry the lab raises, which is what a real node pulls from anyway — what is
|
// registry prefix is the lab's; the digests are upstream's.
|
||||||
// required is a reference that is exact and cannot move (novox/hq ADR 0006).
|
|
||||||
//
|
//
|
||||||
// The store waits up to three minutes rather than one. A machine that has just pulled the
|
// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
|
||||||
// image and is running initdb for the first time can take longer than sixty seconds, and it
|
// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
|
||||||
// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed,
|
// the image's postgres user, its superuser password read from the module's own-secret file — so the
|
||||||
// which is the worst kind because it teaches people to run things twice.
|
// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
|
||||||
//
|
// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
|
||||||
// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited
|
// them to each other against the module's manifest).
|
||||||
// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes
|
|
||||||
// a retry the only available response is a timeout that reports nothing. So the wait now says
|
|
||||||
// what it saw before giving up.
|
|
||||||
//
|
|
||||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
|
||||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
|
||||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
|
||||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
|
||||||
// state.
|
|
||||||
{
|
{
|
||||||
"declaration": 1,
|
"declaration": 1,
|
||||||
"resources": [
|
"resources": [
|
||||||
@@ -80,17 +84,28 @@
|
|||||||
"boot": "enabled",
|
"boot": "enabled",
|
||||||
"restart-on": ["base-filter"]
|
"restart-on": ["base-filter"]
|
||||||
},
|
},
|
||||||
|
// The store's data is a directory owned by the image's postgres user (999), made before the
|
||||||
|
// container starts: a directory the runtime made would be root's, and the database could not
|
||||||
|
// write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
|
||||||
|
// made so (novox/hq ADR 0338 §5).
|
||||||
|
{
|
||||||
|
"id": "store-data",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/mesh-store",
|
||||||
|
"mode": "0700",
|
||||||
|
"owner": "999:70"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "store",
|
"id": "store",
|
||||||
"type": "container",
|
"type": "container",
|
||||||
"name": "mesh-store",
|
"name": "postgres",
|
||||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
"image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
|
||||||
"env": {
|
"env": {
|
||||||
"POSTGRES_PASSWORD": "bootstrap",
|
"POSTGRES_PASSWORD": "bootstrap",
|
||||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||||
},
|
},
|
||||||
"ports": ["5432:5432"],
|
"ports": ["5432:5432"],
|
||||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
"volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
|
||||||
},
|
},
|
||||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||||
@@ -99,21 +114,21 @@
|
|||||||
{
|
{
|
||||||
"id": "store-ready",
|
"id": "store-ready",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"in": "mesh-store",
|
"in": "postgres",
|
||||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "inventory-database",
|
"id": "inventory-database",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"in": "mesh-store",
|
"in": "postgres",
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "identity-database",
|
"id": "identity-database",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"in": "mesh-store",
|
"in": "postgres",
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||||
},
|
},
|
||||||
@@ -123,46 +138,112 @@
|
|||||||
{
|
{
|
||||||
"id": "licences-database",
|
"id": "licences-database",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"in": "mesh-store",
|
"in": "postgres",
|
||||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "context-schemas",
|
"id": "context-schemas",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
"command": ["docker", "run", "--rm", "--network", "container:postgres",
|
||||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
"migrate"],
|
"migrate"],
|
||||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
"verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||||
|
},
|
||||||
|
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||||
|
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||||
|
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
|
||||||
|
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
|
||||||
|
// module's built image takes the place of the upstream one below, once, on the same streams, users
|
||||||
|
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
|
||||||
|
// against the module's own manifest (busLayout), at the broker step and in its tests.
|
||||||
|
//
|
||||||
|
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
|
||||||
|
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
|
||||||
|
// The certificate's directory is readable by everyone because the certificate is public and the control
|
||||||
|
// plane runs as nobody; its key is written 0600 by the program that makes it.
|
||||||
|
{
|
||||||
|
"id": "bus-tls-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/mesh-broker-tls",
|
||||||
|
"mode": "0755"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "broker-certificate",
|
"id": "bus-data-dir",
|
||||||
"type": "action",
|
"type": "directory",
|
||||||
"command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
"path": "/var/lib/mesh-broker-nats",
|
||||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
"mode": "0700"
|
||||||
"-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
|
|
||||||
"verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
|
||||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
|
||||||
"-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
|
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-certificate",
|
||||||
|
"type": "action",
|
||||||
|
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
||||||
|
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
||||||
|
// shell and no openssl, and no other image the bundle names has one either. So the program
|
||||||
|
// that needs the certificate writes it — already on this machine, since the schema step ran
|
||||||
|
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||||
|
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||||
|
// to ask an authority of.
|
||||||
|
// `--user 0:0` because the directory is root's and this image runs as nobody, which is right
|
||||||
|
// for the long-running control plane and wrong for a one-shot writing into a fresh directory.
|
||||||
|
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||||
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
|
"broker", "certificate", "--into", "/tls"],
|
||||||
|
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||||
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
|
"broker", "certificate", "--check", "--into", "/tls"]
|
||||||
|
},
|
||||||
|
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
|
||||||
|
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
|
||||||
|
// run again after the adoption applies this bundle again, and an ordinary file would put the
|
||||||
|
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
|
||||||
|
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
|
||||||
|
{
|
||||||
|
"id": "bus-conf-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/nats-module/conf",
|
||||||
|
"mode": "0700"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-conf",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/var/lib/nats-module/conf/nats.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"create-once": true,
|
||||||
|
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-accounts",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/var/lib/nats-module/conf/accounts.conf",
|
||||||
|
"mode": "0600",
|
||||||
|
"create-once": true,
|
||||||
|
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||||
|
},
|
||||||
|
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
|
||||||
|
// mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
|
||||||
|
// step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17),
|
||||||
|
// pinned upstream by that same digest: the adoption changes no server release, so it can be undone
|
||||||
|
// by putting the old image back and is said reversible. The published port is the
|
||||||
|
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
|
||||||
|
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
|
||||||
{
|
{
|
||||||
"id": "broker",
|
"id": "broker",
|
||||||
"type": "container",
|
"type": "container",
|
||||||
"name": "mesh-broker",
|
"name": "mesh-broker-nats",
|
||||||
"image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
"image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00",
|
||||||
"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"],
|
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||||
"volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"],
|
"volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
|
||||||
"args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"]
|
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "broker-ready",
|
"id": "broker-ready",
|
||||||
"type": "action",
|
"type": "action",
|
||||||
"in": "mesh-broker",
|
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||||
"command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"]
|
||||||
"verify": ["lavinmqctl", "status"]
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "control-plane",
|
"id": "control-plane",
|
||||||
@@ -171,13 +252,12 @@
|
|||||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
"network": "host",
|
"network": "host",
|
||||||
"args": ["serve"],
|
"args": ["serve"],
|
||||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
"volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
|
||||||
"env": {
|
"env": {
|
||||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||||
"MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/",
|
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||||
"MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672",
|
|
||||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,13 +3,52 @@ module github.com/novox/mesh-host
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/klauspost/compress v1.20.0 // indirect
|
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
|
||||||
|
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||||
|
github.com/google/go-tpm v0.9.8 // indirect
|
||||||
|
github.com/klauspost/compress v1.20.1 // indirect
|
||||||
|
github.com/minio/highwayhash v1.0.4 // indirect
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/time v0.16.0 // indirect
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
|
||||||
|
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,15 @@
|
|||||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||||
|
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||||
|
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
|
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||||
|
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
@@ -8,5 +18,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
|||||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||||
|
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||||
|
|||||||
@@ -0,0 +1,320 @@
|
|||||||
|
// Package accounts is the node-engine reading whether an account has, where it runs, the groups a module
|
||||||
|
// put it in (novox/hq ADR 0252, issue 247).
|
||||||
|
//
|
||||||
|
// **A group takes effect at the next login.** The apply puts the account in a group in the machine's
|
||||||
|
// database at once. Every process already running keeps the groups it started with: the account's own
|
||||||
|
// service manager, and every unit it starts. The lighting daemon's module puts the operator's account in
|
||||||
|
// `openrazer`, and the daemon, started by that manager, still refuses to start, for as long as the
|
||||||
|
// session that began before lasts. Nothing said why.
|
||||||
|
//
|
||||||
|
// On every look the engine reads, for each account a module declares groups for:
|
||||||
|
//
|
||||||
|
// 1. whether the user database lists the account in each group. One it does not is the apply's to put
|
||||||
|
// right, and is said as `not in the group`;
|
||||||
|
// 2. whether the account's own service manager runs, and if it does, the groups that process holds,
|
||||||
|
// read from the process itself. A group the database lists and the running manager lacks is said as
|
||||||
|
// **relogin needed**, with what to do.
|
||||||
|
//
|
||||||
|
// No running manager is healthy: nobody is logged in, and the next login takes the groups.
|
||||||
|
//
|
||||||
|
// Each verdict is the declaring module's, as a resource of kind `account`, beside what liveness says: the
|
||||||
|
// controller raises it as the module's condition on two statements in a row, and clears it on the first
|
||||||
|
// healthy one.
|
||||||
|
//
|
||||||
|
// **And an account declared never to become root without a person** (`root: never`, novox/hq ADR 0266) —
|
||||||
|
// the account agent sessions run as on a machine where they must not reach root by themselves — is judged
|
||||||
|
// on every look for whether it can anyway: by its uid, by a group that grants root to its members, by any
|
||||||
|
// sudo rule at all, or by reading a secret the mesh placed for another account (the tool runner's bus
|
||||||
|
// credential among them, which carries every co-hosted module's grants). Judged first, whether or not
|
||||||
|
// anybody is logged in: a way to root does not wait for a session. Any way found is unhealthy, its reason
|
||||||
|
// starting ReasonRoot and naming every way; a question the machine did not answer is unknown, never
|
||||||
|
// healthy. This is the fact the controller reads to tell a machine where no agent can become root from one
|
||||||
|
// where an agent can.
|
||||||
|
//
|
||||||
|
// **It reads; it never acts** (ADR 0240 rule 6): the user and group databases, the machine's own service
|
||||||
|
// manager's `show` of the account's manager unit, and that process's status file. It never starts the
|
||||||
|
// account's manager, which asking that manager itself would.
|
||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The states, in the words liveness says them (the controller reads them alike).
|
||||||
|
const (
|
||||||
|
Healthy = "healthy"
|
||||||
|
Unhealthy = "unhealthy"
|
||||||
|
Unknown = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReasonRelogin starts the reason of an account whose running session lacks a group it is in.
|
||||||
|
const ReasonRelogin = "relogin needed"
|
||||||
|
|
||||||
|
// ReasonRoot starts the reason of an account declared never to become root without a person that can
|
||||||
|
// (novox/hq ADR 0266); every way found follows it.
|
||||||
|
const ReasonRoot = "can become root without a person"
|
||||||
|
|
||||||
|
// RootGroups are the groups whose members become root by membership alone, without a sudo rule naming
|
||||||
|
// them: the administrators' groups a distribution's own rules or polkit treat as root, the container
|
||||||
|
// runtime's socket (a container with the host's root mounted), the raw disks, and the virtualisation
|
||||||
|
// daemons that start a guest with the host's devices — and the kernel's memory (kmem) and the password hashes
|
||||||
|
// (shadow), either of which is root a step away (novox/hq ADR 0312 §4 review). A closed list, so what the judge
|
||||||
|
// calls "grants root" is written down and reviewable rather than guessed per machine.
|
||||||
|
var RootGroups = []string{"root", "wheel", "sudo", "admin", "docker", "disk", "lxd", "incus-admin", "libvirt",
|
||||||
|
"kmem", "shadow"}
|
||||||
|
|
||||||
|
// Account is one user resource of a module that declares groups.
|
||||||
|
type Account struct {
|
||||||
|
Module string
|
||||||
|
// ID is the user resource's id; Name the account.
|
||||||
|
ID string
|
||||||
|
Name string
|
||||||
|
Groups []string
|
||||||
|
// Root is true for an account declared never to become root without a person (novox/hq ADR 0266).
|
||||||
|
Root bool
|
||||||
|
// Secrets are the paths of every secret the declaration places for another account, judged for
|
||||||
|
// whether this one can read them; only for a Root account.
|
||||||
|
Secrets []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Of is every account a declaration has a module put in a group. held is every resource an adopted
|
||||||
|
// machine holds as found, by id: its groups are not the mesh's yet. A resource the mesh declares in its
|
||||||
|
// own right (no module) is not judged here.
|
||||||
|
func Of(d *declaration.Declaration, held map[string]bool) []Account {
|
||||||
|
if d == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []Account
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
u, ok := r.(*declaration.User)
|
||||||
|
root := ok && u.Root == declaration.RootNever
|
||||||
|
if !ok || (len(u.Groups) == 0 && !root) || held[u.ID] || u.Name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := strings.LastIndex(u.ID, ".")
|
||||||
|
if at <= 0 || strings.HasPrefix(u.ID, declaration.AdoptionPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a := Account{Module: u.ID[:at], ID: u.ID, Name: u.Name, Groups: append([]string(nil), u.Groups...), Root: root}
|
||||||
|
if root {
|
||||||
|
a.Secrets = secretsOf(d, u.Name)
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretsOf is the path of every secret file the declaration places for anybody but account: sealed whole,
|
||||||
|
// or with sealed values in its content. One the account owns is its own to read.
|
||||||
|
func secretsOf(d *declaration.Declaration, account string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
f, ok := r.(*declaration.File)
|
||||||
|
if !ok || f.Path == "" || (!f.Secret() && len(f.Secrets) == 0) || f.Owner == account {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, f.Path)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Session is what an account's own service manager holds.
|
||||||
|
type Session struct {
|
||||||
|
// Running is whether the manager runs.
|
||||||
|
Running bool
|
||||||
|
// Has is, of the groups asked about, those the running manager holds.
|
||||||
|
Has map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader is what a look asks the machine. An error is "could not be read": said unknown, never healthy.
|
||||||
|
type Reader interface {
|
||||||
|
// InDatabase is every group the user database lists an account in.
|
||||||
|
InDatabase(ctx context.Context, account string) ([]string, error)
|
||||||
|
// Session is the account's own service manager: whether it runs, and which of groups it holds.
|
||||||
|
Session(ctx context.Context, account string, groups []string) (Session, error)
|
||||||
|
// Escalation is every way the account can become root without a person, in words — its uid, a group
|
||||||
|
// of RootGroups, a sudo rule, a secret of secrets it can read — and none when there is none. Reads
|
||||||
|
// only (novox/hq ADR 0266).
|
||||||
|
Escalation(ctx context.Context, account string, secrets []string) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verdict is one account's state as a statement says it.
|
||||||
|
type Verdict struct {
|
||||||
|
Account
|
||||||
|
State string
|
||||||
|
Reason string
|
||||||
|
Since time.Time
|
||||||
|
Streak int
|
||||||
|
// Search is how far the setuid search got, while the verdict waits for it (novox/hq issue 361); nil
|
||||||
|
// otherwise. Never part of Reason: it moves every minute, and a verdict that changed every minute would
|
||||||
|
// be said in the journal every minute.
|
||||||
|
Search *SearchProgress
|
||||||
|
}
|
||||||
|
|
||||||
|
// Statement is one look at every account.
|
||||||
|
type Statement struct {
|
||||||
|
At time.Time
|
||||||
|
Accounts []Verdict
|
||||||
|
}
|
||||||
|
|
||||||
|
// Healthy says no account in it is anything but healthy.
|
||||||
|
func (s Statement) Healthy() bool {
|
||||||
|
for _, v := range s.Accounts {
|
||||||
|
if v.State != Healthy {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Judge reads every account's groups on every look. Safe for the apply and the looking loop at once.
|
||||||
|
type Judge struct {
|
||||||
|
reader Reader
|
||||||
|
Now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
accounts []Account
|
||||||
|
kept map[string]*Verdict
|
||||||
|
last Statement
|
||||||
|
}
|
||||||
|
|
||||||
|
// New is a judge reading through r.
|
||||||
|
func New(r Reader) *Judge {
|
||||||
|
return &Judge{reader: r, Now: time.Now, kept: map[string]*Verdict{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set is what the declaration just applied asks. An account no longer asked for is forgotten.
|
||||||
|
func (j *Judge) Set(as []Account) {
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
j.accounts = append([]Account(nil), as...)
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, a := range as {
|
||||||
|
declared[a.ID] = true
|
||||||
|
}
|
||||||
|
for id := range j.kept {
|
||||||
|
if !declared[id] {
|
||||||
|
delete(j.kept, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last is the statement of the last look.
|
||||||
|
func (j *Judge) Last() Statement {
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
return j.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look reads every account once, and answers the statement and whether any verdict changed since the
|
||||||
|
// last look.
|
||||||
|
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
now := j.Now()
|
||||||
|
st := Statement{At: now}
|
||||||
|
changed := len(j.last.Accounts) != len(j.accounts)
|
||||||
|
for _, a := range j.accounts {
|
||||||
|
state, reason, search := j.judge(ctx, a)
|
||||||
|
k := j.kept[a.ID]
|
||||||
|
if k == nil || k.State != state || k.Reason != reason {
|
||||||
|
changed = true
|
||||||
|
k = &Verdict{Account: a, State: state, Reason: reason, Since: now}
|
||||||
|
j.kept[a.ID] = k
|
||||||
|
}
|
||||||
|
k.Account, k.Search = a, search
|
||||||
|
if state == Unhealthy {
|
||||||
|
k.Streak++
|
||||||
|
} else {
|
||||||
|
k.Streak = 0
|
||||||
|
}
|
||||||
|
st.Accounts = append(st.Accounts, *k)
|
||||||
|
}
|
||||||
|
j.last = st
|
||||||
|
return st, changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// judge is one account's verdict on one look, and how far the setuid search got while it waits for one.
|
||||||
|
func (j *Judge) judge(ctx context.Context, a Account) (string, string, *SearchProgress) {
|
||||||
|
if a.Root {
|
||||||
|
ways, err := j.reader.Escalation(ctx, a.Name, a.Secrets)
|
||||||
|
var notJudged *SetuidNotJudged
|
||||||
|
switch {
|
||||||
|
case len(ways) > 0:
|
||||||
|
// A way found is a way found, whatever else could not be judged; said beside it.
|
||||||
|
reason := ReasonRoot + ": " + strings.Join(ways, "; ")
|
||||||
|
if errors.As(err, ¬Judged) {
|
||||||
|
reason += "; " + firstLine(err.Error())
|
||||||
|
}
|
||||||
|
var progress *SearchProgress
|
||||||
|
if notJudged != nil {
|
||||||
|
progress = notJudged.Progress
|
||||||
|
}
|
||||||
|
return Unhealthy, reason, progress
|
||||||
|
case errors.As(err, ¬Judged):
|
||||||
|
// The setuid search has no result yet: not judged, said as such — the same words on every look
|
||||||
|
// until it has one, never a search that stalls the look (novox/hq ADR 0266).
|
||||||
|
return Unknown, firstLine(err.Error()), notJudged.Progress
|
||||||
|
case err != nil:
|
||||||
|
return Unknown, "whether the account can become root could not be read: " + firstLine(err.Error()), nil
|
||||||
|
}
|
||||||
|
if len(a.Groups) == 0 {
|
||||||
|
// Declared for root alone: nothing of a session to read.
|
||||||
|
return Healthy, "", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
in, err := j.reader.InDatabase(ctx, a.Name)
|
||||||
|
if err != nil {
|
||||||
|
return Unknown, "the user database could not be read: " + firstLine(err.Error()), nil
|
||||||
|
}
|
||||||
|
listed := map[string]bool{}
|
||||||
|
for _, g := range in {
|
||||||
|
listed[g] = true
|
||||||
|
}
|
||||||
|
var missing, inDB []string
|
||||||
|
for _, g := range a.Groups {
|
||||||
|
if listed[g] {
|
||||||
|
inDB = append(inDB, g)
|
||||||
|
} else {
|
||||||
|
missing = append(missing, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
return Unhealthy, fmt.Sprintf("not in the group %s: the apply has not put %s there; its outcome says why",
|
||||||
|
strings.Join(missing, ", "), a.Name), nil
|
||||||
|
}
|
||||||
|
s, err := j.reader.Session(ctx, a.Name, inDB)
|
||||||
|
if err != nil {
|
||||||
|
return Unknown, "the account's own service manager could not be read: " + firstLine(err.Error()), nil
|
||||||
|
}
|
||||||
|
if !s.Running {
|
||||||
|
// Nobody is logged in, and the account does not linger: the next login takes every group.
|
||||||
|
return Healthy, "", nil
|
||||||
|
}
|
||||||
|
var lacking []string
|
||||||
|
for _, g := range inDB {
|
||||||
|
if !s.Has[g] {
|
||||||
|
lacking = append(lacking, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(lacking) > 0 {
|
||||||
|
return Unhealthy, fmt.Sprintf("%s: %s is in the group %s, and its running session began before it was; "+
|
||||||
|
"log out of every session and in again, or reboot", ReasonRelogin, a.Name, strings.Join(lacking, ", ")), nil
|
||||||
|
}
|
||||||
|
return Healthy, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstLine(s string) string {
|
||||||
|
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||||
|
return line
|
||||||
|
}
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The account judge (novox/hq ADR 0252, "how it is checked"): an account in a group its running session
|
||||||
|
// lacks is unhealthy with "relogin needed"; the same account with no session, or a session that has the
|
||||||
|
// group, is healthy; one the database does not list in the group is unhealthy saying so; and everything
|
||||||
|
// asked of the machine is a read.
|
||||||
|
|
||||||
|
// machine is a fake: the database's groups, the manager's process and its status file.
|
||||||
|
type machine struct {
|
||||||
|
t *testing.T
|
||||||
|
proc string
|
||||||
|
inDB string
|
||||||
|
pid string
|
||||||
|
held string // the Groups line of the manager's status
|
||||||
|
asked []string
|
||||||
|
failsID bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
m.asked = append(m.asked, name+" "+strings.Join(args, " "))
|
||||||
|
switch {
|
||||||
|
case name == "id":
|
||||||
|
if m.failsID {
|
||||||
|
return "", errors.New("id exited 1")
|
||||||
|
}
|
||||||
|
return m.inDB + "\n", nil
|
||||||
|
case name == "getent" && args[0] == "passwd":
|
||||||
|
return "operator:x:1500:1500::/home/operator:/bin/zsh\n", nil
|
||||||
|
case name == "getent" && args[0] == "group":
|
||||||
|
switch args[1] {
|
||||||
|
case "openrazer":
|
||||||
|
return "openrazer:x:964:operator\n", nil
|
||||||
|
case "wheel":
|
||||||
|
return "wheel:x:998:operator\n", nil
|
||||||
|
}
|
||||||
|
return "", errors.New("getent exited 2: ")
|
||||||
|
case name == "systemctl":
|
||||||
|
return m.pid + "\n", nil
|
||||||
|
}
|
||||||
|
return "", errors.New("not a command the judge may run")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *machine) status() {
|
||||||
|
m.t.Helper()
|
||||||
|
if m.pid == "" || m.pid == "0" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dir := filepath.Join(m.proc, m.pid)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
m.t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := "Name:\tsystemd\nUid:\t1500\t1500\t1500\t1500\nGid:\t1500\t1500\t1500\t1500\nGroups:\t" + m.held + "\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "status"), []byte(body), 0o644); err != nil {
|
||||||
|
m.t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var razer = Account{Module: "openrazer", ID: "openrazer.account", Name: "operator", Groups: []string{"openrazer"}}
|
||||||
|
|
||||||
|
func look(t *testing.T, m *machine) Verdict {
|
||||||
|
t.Helper()
|
||||||
|
m.status()
|
||||||
|
j := New(Exec{Run: m.run, Proc: m.proc})
|
||||||
|
j.Now = func() time.Time { return time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) }
|
||||||
|
j.Set([]Account{razer})
|
||||||
|
st, _ := j.Look(t.Context())
|
||||||
|
if len(st.Accounts) != 1 {
|
||||||
|
t.Fatalf("the statement: %+v", st)
|
||||||
|
}
|
||||||
|
return st.Accounts[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGroupTheRunningSessionLacksSaysReloginNeeded(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), inDB: "operator wheel openrazer", pid: "4242", held: "998"}
|
||||||
|
v := look(t, m)
|
||||||
|
if v.State != Unhealthy || !strings.HasPrefix(v.Reason, ReasonRelogin) || !strings.Contains(v.Reason, "openrazer") ||
|
||||||
|
!strings.Contains(v.Reason, "log out") {
|
||||||
|
t.Fatalf("a session without the group: %+v", v)
|
||||||
|
}
|
||||||
|
if v.Module != "openrazer" || v.ID != "openrazer.account" {
|
||||||
|
t.Errorf("the verdict is not the module's: %+v", v)
|
||||||
|
}
|
||||||
|
for _, a := range m.asked {
|
||||||
|
read := strings.HasPrefix(a, "id -nG ") || strings.HasPrefix(a, "getent ") ||
|
||||||
|
strings.HasPrefix(a, "systemctl show --property=MainPID --value user@")
|
||||||
|
if !read {
|
||||||
|
t.Errorf("the judge asked something that is not a read: %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASessionThatHasTheGroupIsHealthy(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), inDB: "operator wheel openrazer", pid: "4242", held: "998 964"}
|
||||||
|
if v := look(t, m); v.State != Healthy || v.Reason != "" {
|
||||||
|
t.Fatalf("a session with the group: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoSessionIsHealthyTheNextLoginTakesTheGroup(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), inDB: "operator openrazer", pid: "0"}
|
||||||
|
if v := look(t, m); v.State != Healthy {
|
||||||
|
t.Fatalf("no session: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAccountTheDatabaseDoesNotListIsUnhealthySayingSo(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), inDB: "operator wheel", pid: "4242", held: "998"}
|
||||||
|
v := look(t, m)
|
||||||
|
if v.State != Unhealthy || !strings.HasPrefix(v.Reason, "not in the group openrazer") {
|
||||||
|
t.Fatalf("not in the group: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADatabaseThatDoesNotAnswerIsUnknownNeverHealthy(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), failsID: true}
|
||||||
|
if v := look(t, m); v.State != Unknown {
|
||||||
|
t.Fatalf("an unread database: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyAModulesAccountWithGroupsIsJudged(t *testing.T) {
|
||||||
|
d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"zsh.login","type":"user","name":"operator","shell":"/bin/zsh"},
|
||||||
|
{"id":"openrazer.account","type":"user","name":"operator","groups":["openrazer"]},
|
||||||
|
{"id":"docker.account","type":"user","name":"operator","groups":["docker"]},
|
||||||
|
{"id":"meshown","type":"user","name":"mesh","groups":["x"]}
|
||||||
|
]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := Of(d, map[string]bool{"docker.account": true})
|
||||||
|
if len(got) != 1 || got[0].ID != "openrazer.account" || got[0].Module != "openrazer" {
|
||||||
|
t.Fatalf("judged: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAVerdictThatHoldsCountsItsStreakAndAChangeIsSaid(t *testing.T) {
|
||||||
|
m := &machine{t: t, proc: t.TempDir(), inDB: "operator openrazer", pid: "4242", held: "998"}
|
||||||
|
m.status()
|
||||||
|
j := New(Exec{Run: m.run, Proc: m.proc})
|
||||||
|
j.Set([]Account{razer})
|
||||||
|
_, changed := j.Look(t.Context())
|
||||||
|
st, again := j.Look(t.Context())
|
||||||
|
if !changed || again || st.Accounts[0].Streak != 2 {
|
||||||
|
t.Fatalf("first look changed %v, second %v, streak %d", changed, again, st.Accounts[0].Streak)
|
||||||
|
}
|
||||||
|
m.held = "998 964"
|
||||||
|
m.status()
|
||||||
|
st, changed = j.Look(t.Context())
|
||||||
|
if !changed || st.Accounts[0].State != Healthy || st.Accounts[0].Streak != 0 {
|
||||||
|
t.Fatalf("after the new login: changed %v, %+v", changed, st.Accounts[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,316 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner).
|
||||||
|
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||||
|
|
||||||
|
// Exec reads the machine through its command lines and the process table. **Reads only**: `id`, `getent`,
|
||||||
|
// the machine's own manager's `systemctl show`, `sudo -l -U` (which lists, as root, what sudo would let an
|
||||||
|
// account run, and runs nothing), a status file under /proc and a secret's owner and mode (a test holds
|
||||||
|
// both).
|
||||||
|
type Exec struct {
|
||||||
|
Run Runner
|
||||||
|
// Proc is where the process table is; empty is /proc. A test points it at a directory of its own.
|
||||||
|
Proc string
|
||||||
|
// Stat is a file's owner and mode; nil is the machine's own (os.Stat). A test gives files of its own.
|
||||||
|
Stat func(path string) (FileMode, error)
|
||||||
|
// Lstat is a path's own owner and mode, never through a link, with its type bits in Perm (fs.ModeSymlink,
|
||||||
|
// fs.ModeDir); nil is the machine's own (os.Lstat). Whether an account can reach a setuid program is read
|
||||||
|
// with it at every judgement (novox/hq issue 361).
|
||||||
|
Lstat func(path string) (FileMode, error)
|
||||||
|
// ReadFile, ReadDir and ACL read doas's and polkit's rules and a file's POSIX ACL; nil is the machine's.
|
||||||
|
ReadFile func(path string) ([]byte, error)
|
||||||
|
ReadDir func(path string) ([]fs.DirEntry, error)
|
||||||
|
ACL func(path string) ([]ACLEntry, error)
|
||||||
|
// Cache keeps the search for setuid programs between looks; nil searches on every look.
|
||||||
|
Cache *SetuidCache
|
||||||
|
// SearchRun runs the setuid search's commands (the package manager it asks); nil is Run. A
|
||||||
|
// serving engine gives Idle, so they yield to everything else on the machine, as the walk does.
|
||||||
|
SearchRun Runner
|
||||||
|
// Walk carries a walk for setuid programs on; nil is the engine's own (walker). A test gives its own.
|
||||||
|
Walk func(ctx context.Context, p *SetuidProgress, checkpoint func()) error
|
||||||
|
// Now is the clock the cache is kept by; nil is the machine's.
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMode is what decides whether an account reads a file: its owner, its group and its permission bits.
|
||||||
|
type FileMode struct {
|
||||||
|
UID, GID int
|
||||||
|
Perm fs.FileMode
|
||||||
|
}
|
||||||
|
|
||||||
|
// Escalation is every way account can become root without a person (novox/hq ADR 0266) that the judge
|
||||||
|
// looks for — Judged lists them, NotJudged what it does not: its uid, a group of RootGroups the user database
|
||||||
|
// lists it in, any sudo rule naming it or a group of it, any of secrets it can read by owner, group or other
|
||||||
|
// bits, and the ways of ways.go: doas, polkit, a runtime's socket, an ACL, a setuid program no package owns.
|
||||||
|
// sudo absent is no sudo rule; doas and polkit (pkexec's grants) are judged from their own rules. A secret not there yet is skipped: there is nothing to read.
|
||||||
|
// The parent directories are not walked, so a file the bits allow and a directory hides is still said:
|
||||||
|
// the judge errs toward saying a way that is not, never toward missing one that is.
|
||||||
|
func (e Exec) Escalation(ctx context.Context, account string, secrets []string) ([]string, error) {
|
||||||
|
var ways []string
|
||||||
|
uidOut, err := e.Run(ctx, "id", "-u", account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the user database did not answer about %q: %w", account, err)
|
||||||
|
}
|
||||||
|
uid, err := strconv.Atoi(strings.TrimSpace(uidOut))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the user database gave %q as %q's number", strings.TrimSpace(uidOut), account)
|
||||||
|
}
|
||||||
|
if uid == 0 {
|
||||||
|
ways = append(ways, "its uid is 0")
|
||||||
|
}
|
||||||
|
names, err := e.InDatabase(ctx, account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, g := range names {
|
||||||
|
if slices.Contains(RootGroups, g) {
|
||||||
|
ways = append(ways, "in the group "+g+", which grants root")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
listed, err := e.Run(ctx, "sudo", "-l", "-U", account)
|
||||||
|
rules, err := SudoRules(listed, err)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(rules) > 0 {
|
||||||
|
ways = append(ways, "sudo grants it: "+strings.Join(rules, ", "))
|
||||||
|
}
|
||||||
|
gidsOut, err := e.Run(ctx, "id", "-G", account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the user database did not answer about %q's groups: %w", account, err)
|
||||||
|
}
|
||||||
|
gids := map[int]bool{}
|
||||||
|
for _, f := range strings.Fields(gidsOut) {
|
||||||
|
if n, err := strconv.Atoi(f); err == nil {
|
||||||
|
gids[n] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(secrets) > 0 {
|
||||||
|
stat := e.Stat
|
||||||
|
if stat == nil {
|
||||||
|
stat = statOf
|
||||||
|
}
|
||||||
|
for _, path := range secrets {
|
||||||
|
m, err := stat(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the secret %s could not be read for its owner and mode: %w", path, err)
|
||||||
|
}
|
||||||
|
if Readable(m, uid, gids) {
|
||||||
|
ways = append(ways, "it can read the secret "+path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
more, err := e.moreWays(ctx, account, uid, names, gids, secrets)
|
||||||
|
var notJudged *SetuidNotJudged
|
||||||
|
if err != nil && !errors.As(err, ¬Judged) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Every way found, and — when the setuid search has no result yet — that it is not judged.
|
||||||
|
return append(ways, more...), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Readable is whether an account of uid, in the groups gids, reads a file of m by its permission bits, as
|
||||||
|
// the kernel decides it: the owner's bits for the owner (root reads everything), the group's for a member,
|
||||||
|
// the others' for anybody else.
|
||||||
|
func Readable(m FileMode, uid int, gids map[int]bool) bool {
|
||||||
|
switch {
|
||||||
|
case uid == 0:
|
||||||
|
return true
|
||||||
|
case m.UID == uid:
|
||||||
|
return m.Perm&0o400 != 0
|
||||||
|
case gids[m.GID]:
|
||||||
|
return m.Perm&0o040 != 0
|
||||||
|
default:
|
||||||
|
return m.Perm&0o004 != 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SudoRules is the rules `sudo -l -U <account>` lists, from what it printed and how it ended: none when
|
||||||
|
// the account "is not allowed to run sudo" or sudo is not on the machine; every indented line after "may
|
||||||
|
// run the following commands" otherwise. Any rule counts — the decision is no sudo for the account at
|
||||||
|
// all, so a rule that asks for a password the account was never given is still a rule somebody can give
|
||||||
|
// it one for. Output that says neither is an error: unread is never none.
|
||||||
|
func SudoRules(out string, err error) ([]string, error) {
|
||||||
|
if err != nil && (errors.Is(err, exec.ErrNotFound) || errors.Is(err, fs.ErrNotExist)) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
text := out
|
||||||
|
if err != nil {
|
||||||
|
text += "\n" + err.Error()
|
||||||
|
}
|
||||||
|
if strings.Contains(text, "is not allowed to run sudo") {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("sudo did not list the account's rules: %w", err)
|
||||||
|
}
|
||||||
|
var rules []string
|
||||||
|
listing := false
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
if strings.Contains(line, "may run the following commands") {
|
||||||
|
listing = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if listing && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) && strings.TrimSpace(line) != "" {
|
||||||
|
rules = append(rules, strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !listing {
|
||||||
|
return nil, fmt.Errorf("sudo listed neither rules nor a refusal: %q", firstLine(out))
|
||||||
|
}
|
||||||
|
return rules, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func statOf(path string) (FileMode, error) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return FileMode{}, err
|
||||||
|
}
|
||||||
|
st, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
return FileMode{}, fmt.Errorf("%s has no owner this platform reports", path)
|
||||||
|
}
|
||||||
|
return FileMode{UID: int(st.Uid), GID: int(st.Gid), Perm: info.Mode().Perm()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lstatOf is a path's own owner and mode, its type bits kept.
|
||||||
|
func lstatOf(path string) (FileMode, error) {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return FileMode{}, err
|
||||||
|
}
|
||||||
|
st, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
return FileMode{}, fmt.Errorf("%s has no owner this platform reports", path)
|
||||||
|
}
|
||||||
|
return FileMode{UID: int(st.Uid), GID: int(st.Gid), Perm: info.Mode()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InDatabase is `id -nG`: every group the user database lists the account in.
|
||||||
|
func (e Exec) InDatabase(ctx context.Context, account string) ([]string, error) {
|
||||||
|
out, err := e.Run(ctx, "id", "-nG", account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return strings.Fields(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Session reads the account's own manager, user@<uid>.service, from the machine's manager — never from
|
||||||
|
// the account's, which asking would start — and the groups its process holds, from its status file.
|
||||||
|
func (e Exec) Session(ctx context.Context, account string, groups []string) (Session, error) {
|
||||||
|
passwd, err := e.Run(ctx, "getent", "passwd", account)
|
||||||
|
if err != nil {
|
||||||
|
return Session{}, fmt.Errorf("the user database did not answer about %q: %w", account, err)
|
||||||
|
}
|
||||||
|
fields := strings.Split(strings.TrimSpace(passwd), ":")
|
||||||
|
if len(fields) < 7 || fields[2] == "" {
|
||||||
|
return Session{}, fmt.Errorf("the user database gave no number for %q", account)
|
||||||
|
}
|
||||||
|
shown, err := e.Run(ctx, "systemctl", "show", "--property=MainPID", "--value", "user@"+fields[2]+".service")
|
||||||
|
if err != nil {
|
||||||
|
return Session{}, fmt.Errorf("the machine's service manager did not say whether %q's own runs: %w", account, err)
|
||||||
|
}
|
||||||
|
pid := strings.TrimSpace(shown)
|
||||||
|
if pid == "" || pid == "0" {
|
||||||
|
return Session{}, nil
|
||||||
|
}
|
||||||
|
held, err := e.heldBy(pid)
|
||||||
|
if err != nil {
|
||||||
|
return Session{}, err
|
||||||
|
}
|
||||||
|
s := Session{Running: true, Has: map[string]bool{}}
|
||||||
|
for _, g := range groups {
|
||||||
|
entry, err := e.Run(ctx, "getent", "group", g)
|
||||||
|
if err != nil {
|
||||||
|
return Session{}, fmt.Errorf("the group database did not answer about %q: %w", g, err)
|
||||||
|
}
|
||||||
|
parts := strings.Split(strings.TrimSpace(entry), ":")
|
||||||
|
if len(parts) < 3 {
|
||||||
|
return Session{}, fmt.Errorf("the group database gave %q for %q, which is not a group entry", entry, g)
|
||||||
|
}
|
||||||
|
s.Has[g] = held[parts[2]]
|
||||||
|
}
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldBy is every group id a process holds, from the Groups line of its status file.
|
||||||
|
func (e Exec) heldBy(pid string) (map[string]bool, error) {
|
||||||
|
proc := e.Proc
|
||||||
|
if proc == "" {
|
||||||
|
proc = "/proc"
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(proc, pid, "status"))
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("the account's manager, process %s, ended while it was read", pid)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Its supplementary groups, and its own group, which the supplementary list need not repeat.
|
||||||
|
held := map[string]bool{}
|
||||||
|
named := false
|
||||||
|
for _, line := range strings.Split(string(raw), "\n") {
|
||||||
|
if rest, ok := strings.CutPrefix(line, "Groups:"); ok {
|
||||||
|
named = true
|
||||||
|
for _, gid := range strings.Fields(rest) {
|
||||||
|
held[gid] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rest, ok := strings.CutPrefix(line, "Gid:"); ok {
|
||||||
|
if f := strings.Fields(rest); len(f) > 0 {
|
||||||
|
held[f[0]] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !named {
|
||||||
|
return nil, fmt.Errorf("process %s's status names no groups", pid)
|
||||||
|
}
|
||||||
|
return held, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IDs is `getent passwd`: an account's uid and primary gid, and whether the user database names it at all. A
|
||||||
|
// database that does not answer is an error, never "no such account".
|
||||||
|
func (e Exec) IDs(ctx context.Context, account string) (uid, gid int, found bool, err error) {
|
||||||
|
out, err := e.Run(ctx, "getent", "passwd", account)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "exited 2") || strings.Contains(err.Error(), "exit status 2") {
|
||||||
|
return 0, 0, false, nil
|
||||||
|
}
|
||||||
|
return 0, 0, false, fmt.Errorf("the user database did not answer about %q: %w", account, err)
|
||||||
|
}
|
||||||
|
fields := strings.Split(strings.TrimSpace(out), ":")
|
||||||
|
if len(fields) < 4 {
|
||||||
|
return 0, 0, false, fmt.Errorf("the user database gave %q for %q, which is not a passwd entry",
|
||||||
|
strings.TrimSpace(out), account)
|
||||||
|
}
|
||||||
|
// getent reads an all-digit key as a uid: "1000" answers with whoever has uid 1000. Only the account named is
|
||||||
|
// the one asked for.
|
||||||
|
if fields[0] != account {
|
||||||
|
return 0, 0, false, fmt.Errorf("the user database answered %q with the account %q: an account is named, "+
|
||||||
|
"never numbered", account, fields[0])
|
||||||
|
}
|
||||||
|
uid, uerr := strconv.Atoi(fields[2])
|
||||||
|
gid, gerr := strconv.Atoi(fields[3])
|
||||||
|
if uerr != nil || gerr != nil {
|
||||||
|
return 0, 0, false, fmt.Errorf("the user database gave no numbers for %q", account)
|
||||||
|
}
|
||||||
|
return uid, gid, true, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ioprioIdle is the idle I/O class (IOPRIO_CLASS_IDLE, 3, shifted by IOPRIO_CLASS_SHIFT, 13): the process gets
|
||||||
|
// the disk only when nothing else asks for it, where the disk's scheduler honours classes (BFQ does).
|
||||||
|
const ioprioIdle = 3 << 13
|
||||||
|
|
||||||
|
// Idle runs a command as CLocale does, at the lowest priority the machine has (novox/hq issue 361): CPU nice
|
||||||
|
// 19 and the idle I/O class, set on the process as soon as it has started — so the search for setuid programs,
|
||||||
|
// which walks every file and runs as long as that takes, yields to everything else. It dies with the engine
|
||||||
|
// (the parent-death signal), so a restarted engine never leaves a walk running beside its own. A priority it
|
||||||
|
// could not lower is a failure: the command is killed rather than walk the disks at full priority.
|
||||||
|
func Idle(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C", "LANGUAGE=C")
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||||
|
// The parent-death signal follows the thread that started the child, not the process: held on this
|
||||||
|
// thread until the child has ended, so the runtime retiring a thread never kills a walk midway.
|
||||||
|
runtime.LockOSThread()
|
||||||
|
defer runtime.UnlockOSThread()
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %w", name, err)
|
||||||
|
}
|
||||||
|
pid := cmd.Process.Pid
|
||||||
|
if err := lower(pid); err != nil {
|
||||||
|
_ = cmd.Process.Kill()
|
||||||
|
_ = cmd.Wait()
|
||||||
|
return "", fmt.Errorf("%s: its priority could not be lowered, so it was stopped: %w", name, err)
|
||||||
|
}
|
||||||
|
err := cmd.Wait()
|
||||||
|
out := stdout.String()
|
||||||
|
if err != nil {
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if errors.As(err, &exit) {
|
||||||
|
return out, fmt.Errorf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
return out, fmt.Errorf("%s: %w", name, err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lower sets a process's CPU nice to 19 and its I/O class to idle.
|
||||||
|
func lower(pid int) error {
|
||||||
|
if err := unix.Setpriority(unix.PRIO_PROCESS, pid, 19); err != nil {
|
||||||
|
return fmt.Errorf("nice: %w", err)
|
||||||
|
}
|
||||||
|
if _, _, errno := unix.Syscall(unix.SYS_IOPRIO_SET, 1 /* IOPRIO_WHO_PROCESS */, uintptr(pid), ioprioIdle); errno != 0 {
|
||||||
|
return fmt.Errorf("I/O class: %w", errno)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LowerThisThread puts the calling goroutine on a thread of its own at nice 19 and the idle I/O class, for the
|
||||||
|
// walk for setuid programs, which runs in the engine (novox/hq issue 361): on Linux both are a thread's, so the
|
||||||
|
// engine's other work keeps its priority. The goroutine is locked to the thread and never unlocked, so the
|
||||||
|
// runtime ends the thread with it rather than hand a lowered thread to other work; a command it starts inherits
|
||||||
|
// the priority too.
|
||||||
|
func LowerThisThread() error {
|
||||||
|
runtime.LockOSThread()
|
||||||
|
return lower(unix.Gettid())
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
//go:build !linux
|
||||||
|
|
||||||
|
package accounts
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// Idle is CLocale where the machine offers no idle class this engine sets (novox/hq issue 361).
|
||||||
|
func Idle(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
return CLocale(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LowerThisThread is nothing where the machine offers no per-thread priority this engine sets.
|
||||||
|
func LowerThisThread() error { return nil }
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An account is read by its name (novox/hq ADR 0325 §4): getent reads an all-digit key as a uid, so "1000" would
|
||||||
|
// answer with whoever has uid 1000 — refused, never taken for the account named.
|
||||||
|
func TestAnAccountIsReadByItsNameNeverItsNumber(t *testing.T) {
|
||||||
|
db := map[string]string{
|
||||||
|
"nox": "nox:x:1003:1003::/home/nox:/bin/sh",
|
||||||
|
"1000": "jochen:x:1000:1000::/home/jochen:/bin/zsh", // what getent answers for a uid
|
||||||
|
}
|
||||||
|
e := Exec{Run: func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
if line, ok := db[args[len(args)-1]]; ok {
|
||||||
|
return line + "\n", nil
|
||||||
|
}
|
||||||
|
return "", errors.New("getent exited 2")
|
||||||
|
}}
|
||||||
|
if uid, gid, found, err := e.IDs(context.Background(), "nox"); err != nil || !found || uid != 1003 || gid != 1003 {
|
||||||
|
t.Fatalf("nox: %d %d %v %v", uid, gid, found, err)
|
||||||
|
}
|
||||||
|
if _, _, found, err := e.IDs(context.Background(), "1000"); found || err == nil || !strings.Contains(err.Error(), "jochen") {
|
||||||
|
t.Fatalf("an all-digit name was read as a uid: found=%v err=%v", found, err)
|
||||||
|
}
|
||||||
|
if _, _, found, err := e.IDs(context.Background(), "ghost"); found || err != nil {
|
||||||
|
t.Fatalf("an absent account: found=%v err=%v", found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An account declared never to become root without a person (novox/hq ADR 0266, "how it is checked"): each
|
||||||
|
// way to root is found and said, none is healthy whether or not anybody is logged in, an unanswered question
|
||||||
|
// is unknown, and the judge only reads.
|
||||||
|
|
||||||
|
// agentMachine is a fake machine for the escalation question: the account's uid, its groups by name and
|
||||||
|
// number, what sudo lists, and the secrets' owners and modes.
|
||||||
|
type agentMachine struct {
|
||||||
|
uid string
|
||||||
|
groups string
|
||||||
|
gids string
|
||||||
|
sudo string
|
||||||
|
sudoErr error
|
||||||
|
files map[string]FileMode
|
||||||
|
failsID bool
|
||||||
|
asked []string
|
||||||
|
// texts are files' contents (doas's, polkit's); dirs a directory's files by name; acls a file's ACL;
|
||||||
|
// setuid what the walk finds, one per line, and packaged the paths a package owns.
|
||||||
|
texts map[string]string
|
||||||
|
dirs map[string][]string
|
||||||
|
acls map[string][]ACLEntry
|
||||||
|
setuid string
|
||||||
|
findErr error
|
||||||
|
packaged map[string]bool
|
||||||
|
// proc is /proc's files, aSafeProc unless a test changes them; findArgs sees the filesystems walked.
|
||||||
|
proc map[string]string
|
||||||
|
findArgs func([]string)
|
||||||
|
// pacmanCalls counts the package manager's calls: one for every path found.
|
||||||
|
pacmanCalls int
|
||||||
|
// nodes are paths' own owners and modes (Lstat); any other is root's, a directory searchable by all, or a
|
||||||
|
// file the walk found, setuid and executable by all.
|
||||||
|
nodes map[string]FileMode
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *agentMachine) lstat(path string) (FileMode, error) {
|
||||||
|
if n, ok := m.nodes[path]; ok {
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
if strings.Contains(m.setuid, path+"\n") {
|
||||||
|
return FileMode{Perm: fs.ModeSetuid | 0o755}, nil
|
||||||
|
}
|
||||||
|
return FileMode{Perm: fs.ModeDir | 0o755}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *agentMachine) readFile(path string) ([]byte, error) {
|
||||||
|
if t, ok := m.texts[path]; ok {
|
||||||
|
return []byte(t), nil
|
||||||
|
}
|
||||||
|
if t, ok := m.proc[path]; ok {
|
||||||
|
return []byte(t), nil
|
||||||
|
}
|
||||||
|
return nil, fs.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
// aSafeProc is what a machine with nothing to say has in /proc: links protected, nothing on the runtimes'
|
||||||
|
// ports, one root filesystem.
|
||||||
|
func aSafeProc() map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
"/proc/sys/fs/protected_hardlinks": "1\n",
|
||||||
|
"/proc/sys/fs/protected_symlinks": "1\n",
|
||||||
|
"/proc/net/tcp": " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" +
|
||||||
|
" 0: 0100007F:1092 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 1 1\n",
|
||||||
|
"/proc/mounts": "/dev/sda2 / ext4 rw,relatime 0 0\nproc /proc proc rw,nosuid 0 0\ntmpfs /tmp tmpfs rw,nosuid 0 0\n",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *agentMachine) readDir(path string) ([]fs.DirEntry, error) {
|
||||||
|
names, ok := m.dirs[path]
|
||||||
|
if !ok {
|
||||||
|
return nil, fs.ErrNotExist
|
||||||
|
}
|
||||||
|
var out []fs.DirEntry
|
||||||
|
for _, n := range names {
|
||||||
|
out = append(out, fakeEntry(n))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeEntry string
|
||||||
|
|
||||||
|
func (f fakeEntry) Name() string { return string(f) }
|
||||||
|
func (f fakeEntry) IsDir() bool { return false }
|
||||||
|
func (f fakeEntry) Type() fs.FileMode { return 0 }
|
||||||
|
func (f fakeEntry) Info() (fs.FileInfo, error) { return nil, fs.ErrNotExist }
|
||||||
|
|
||||||
|
func (m *agentMachine) acl(path string) ([]ACLEntry, error) {
|
||||||
|
if _, ok := m.files[path]; !ok {
|
||||||
|
return nil, fs.ErrNotExist
|
||||||
|
}
|
||||||
|
return m.acls[path], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// walk is the walk for setuid programs on this machine: m.setuid found, then m.findErr.
|
||||||
|
func (m *agentMachine) walk(_ context.Context, p *SetuidProgress, _ func()) error {
|
||||||
|
m.asked = append(m.asked, "walk "+strings.Join(p.Mounts, " "))
|
||||||
|
if m.findArgs != nil {
|
||||||
|
m.findArgs(p.Mounts)
|
||||||
|
}
|
||||||
|
for _, l := range strings.Split(m.setuid, "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
|
p.Found = append(p.Found, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m.findErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *agentMachine) run(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
line := name + " " + strings.Join(args, " ")
|
||||||
|
m.asked = append(m.asked, line)
|
||||||
|
switch {
|
||||||
|
case m.failsID && name == "id":
|
||||||
|
return "", errors.New("id exited 1: no such user")
|
||||||
|
case line == "id -u agent":
|
||||||
|
return m.uid + "\n", nil
|
||||||
|
case line == "id -nG agent":
|
||||||
|
return m.groups + "\n", nil
|
||||||
|
case line == "id -G agent":
|
||||||
|
return m.gids + "\n", nil
|
||||||
|
case line == "sudo -l -U agent":
|
||||||
|
return m.sudo, m.sudoErr
|
||||||
|
case name == "pacman" && len(args) >= 2 && args[0] == "-Qo":
|
||||||
|
m.pacmanCalls++
|
||||||
|
var out, errs []string
|
||||||
|
for _, p := range args[1:] {
|
||||||
|
if m.packaged[p] {
|
||||||
|
out = append(out, p+" is owned by somepackage 1.0-1")
|
||||||
|
} else {
|
||||||
|
errs = append(errs, "error: No package owns "+p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
said := strings.Join(out, "\n")
|
||||||
|
if len(errs) > 0 {
|
||||||
|
return said, errors.New("pacman exited 1: " + strings.Join(errs, "\n"))
|
||||||
|
}
|
||||||
|
return said, nil
|
||||||
|
}
|
||||||
|
return "", errors.New("not a command the judge may run: " + line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *agentMachine) stat(path string) (FileMode, error) {
|
||||||
|
if f, ok := m.files[path]; ok {
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
return FileMode{}, fs.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
const notAllowed = "User agent is not allowed to run sudo on box.\n"
|
||||||
|
|
||||||
|
var agent = Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true,
|
||||||
|
Secrets: []string{"/var/lib/mesh/node-tools/broker"}}
|
||||||
|
|
||||||
|
func clean() *agentMachine {
|
||||||
|
return &agentMachine{uid: "1600", groups: "agent", gids: "1600", sudo: notAllowed,
|
||||||
|
files: map[string]FileMode{"/var/lib/mesh/node-tools/broker": {UID: 1500, GID: 1500, Perm: 0o600}},
|
||||||
|
proc: aSafeProc()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lookAgent(t *testing.T, m *agentMachine) Verdict {
|
||||||
|
t.Helper()
|
||||||
|
j := New(Exec{Run: m.run, Walk: m.walk, Stat: m.stat, ReadFile: m.readFile, ReadDir: m.readDir, ACL: m.acl, Lstat: m.lstat})
|
||||||
|
j.Set([]Account{agent})
|
||||||
|
st, _ := j.Look(t.Context())
|
||||||
|
if len(st.Accounts) != 1 {
|
||||||
|
t.Fatalf("the statement: %+v", st)
|
||||||
|
}
|
||||||
|
for _, a := range m.asked {
|
||||||
|
if !strings.HasPrefix(a, "id ") && a != "sudo -l -U agent" && !strings.HasPrefix(a, "walk /") &&
|
||||||
|
!strings.HasPrefix(a, "pacman -Qo ") {
|
||||||
|
t.Errorf("the judge asked something that is not a read: %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return st.Accounts[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAgentAccountWithNoWayToRootIsHealthyWithNobodyLoggedIn(t *testing.T) {
|
||||||
|
if v := lookAgent(t, clean()); v.State != Healthy || v.Reason != "" || !v.Root {
|
||||||
|
t.Fatalf("no way to root: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEachWayToRootIsSaid(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
set func(*agentMachine)
|
||||||
|
said string
|
||||||
|
}{
|
||||||
|
{"uid 0", func(m *agentMachine) { m.uid = "0" }, "its uid is 0"},
|
||||||
|
{"docker", func(m *agentMachine) { m.groups = "agent docker" }, "in the group docker, which grants root"},
|
||||||
|
{"wheel", func(m *agentMachine) { m.groups = "agent wheel" }, "in the group wheel, which grants root"},
|
||||||
|
{"sudo", func(m *agentMachine) {
|
||||||
|
m.sudo = "Matching Defaults entries for agent on box:\n env_reset\n\nUser agent may run the following commands on box:\n (ALL) NOPASSWD: ALL\n"
|
||||||
|
}, "sudo grants it: (ALL) NOPASSWD: ALL"},
|
||||||
|
{"secret by others", func(m *agentMachine) {
|
||||||
|
m.files["/var/lib/mesh/node-tools/broker"] = FileMode{UID: 1500, GID: 1500, Perm: 0o644}
|
||||||
|
}, "it can read the secret /var/lib/mesh/node-tools/broker"},
|
||||||
|
{"secret by group", func(m *agentMachine) {
|
||||||
|
m.gids = "1600 1500"
|
||||||
|
m.files["/var/lib/mesh/node-tools/broker"] = FileMode{UID: 1500, GID: 1500, Perm: 0o640}
|
||||||
|
}, "it can read the secret /var/lib/mesh/node-tools/broker"},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
m := clean()
|
||||||
|
c.set(m)
|
||||||
|
v := lookAgent(t, m)
|
||||||
|
if v.State != Unhealthy || !strings.HasPrefix(v.Reason, ReasonRoot+": ") || !strings.Contains(v.Reason, c.said) {
|
||||||
|
t.Fatalf("want %q said: %+v", c.said, v)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryWayIsSaidAtOnce(t *testing.T) {
|
||||||
|
m := clean()
|
||||||
|
m.groups = "agent docker"
|
||||||
|
m.sudo = "User agent may run the following commands on box:\n (ALL) ALL\n"
|
||||||
|
v := lookAgent(t, m)
|
||||||
|
if !strings.Contains(v.Reason, "docker") || !strings.Contains(v.Reason, "(ALL) ALL") {
|
||||||
|
t.Fatalf("both ways: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUnansweredQuestionIsUnknownNeverHealthy(t *testing.T) {
|
||||||
|
m := clean()
|
||||||
|
m.failsID = true
|
||||||
|
if v := lookAgent(t, m); v.State != Unknown {
|
||||||
|
t.Fatalf("an unread database: %+v", v)
|
||||||
|
}
|
||||||
|
m = clean()
|
||||||
|
m.sudo = "something sudo never says\n"
|
||||||
|
if v := lookAgent(t, m); v.State != Unknown {
|
||||||
|
t.Fatalf("an unread sudo: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASecretNotThereYetIsNoWay(t *testing.T) {
|
||||||
|
m := clean()
|
||||||
|
delete(m.files, "/var/lib/mesh/node-tools/broker")
|
||||||
|
if v := lookAgent(t, m); v.State != Healthy {
|
||||||
|
t.Fatalf("no secret placed: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSudoRules(t *testing.T) {
|
||||||
|
none := []struct {
|
||||||
|
out string
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{notAllowed, nil},
|
||||||
|
{notAllowed, errors.New("sudo exited 1: ")},
|
||||||
|
{"", fmt.Errorf("sudo: %w", exec.ErrNotFound)},
|
||||||
|
}
|
||||||
|
for _, c := range none {
|
||||||
|
if rules, err := SudoRules(c.out, c.err); err != nil || len(rules) != 0 {
|
||||||
|
t.Errorf("%q, %v: rules %v, err %v", c.out, c.err, rules, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rules, err := SudoRules("Matching Defaults entries for agent on box:\n env_reset\n\nUser agent may run the following commands on box:\n (ALL) /usr/bin/systemctl\n (root) NOPASSWD: /usr/bin/true\n", nil)
|
||||||
|
if err != nil || len(rules) != 2 || rules[0] != "(ALL) /usr/bin/systemctl" {
|
||||||
|
t.Fatalf("two rules: %v, %v", rules, err)
|
||||||
|
}
|
||||||
|
if _, err := SudoRules("", errors.New("sudo exited 1: sudo: unable to resolve host")); err == nil {
|
||||||
|
t.Error("a failing sudo that said neither was read as no rules")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadable(t *testing.T) {
|
||||||
|
m := FileMode{UID: 1500, GID: 1500, Perm: 0o600}
|
||||||
|
if Readable(m, 1600, map[int]bool{1600: true}) || !Readable(m, 1500, nil) || !Readable(m, 0, nil) {
|
||||||
|
t.Fatal("owner bits")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOfJudgesARootNeverAccountWithNoGroupsAndItsSecrets(t *testing.T) {
|
||||||
|
d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"claude-code.agent","type":"user","name":"agent","root":"never"},
|
||||||
|
{"id":"zsh.login","type":"user","name":"operator","shell":"/bin/zsh"},
|
||||||
|
{"id":"node-tools.need-broker","type":"file","path":"/var/lib/mesh/broker","sealed":"x","owner":"operator"},
|
||||||
|
{"id":"claude-code.own","type":"file","path":"/home/agent/own","sealed":"x","owner":"agent"},
|
||||||
|
{"id":"claude-code.plain","type":"file","path":"/etc/plain","content":"x"}
|
||||||
|
]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := Of(d, nil)
|
||||||
|
if len(got) != 1 || got[0].ID != "claude-code.agent" || !got[0].Root ||
|
||||||
|
len(got[0].Secrets) != 1 || got[0].Secrets[0] != "/var/lib/mesh/broker" {
|
||||||
|
t.Fatalf("judged: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,646 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/rootsearch"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SetuidCache keeps the search for setuid programs, which walks every local filesystem, and runs it in the
|
||||||
|
// background (novox/hq ADR 0266, issue 361): a look never waits for it, and the search itself is never cut
|
||||||
|
// short.
|
||||||
|
//
|
||||||
|
// - **A verdict comes only from a complete search, while it is fresh**: one that walked everything and had the
|
||||||
|
// package manager answer for every path found, started less than FreshFor ago. Before the first completes,
|
||||||
|
// or once the last is older, the answer is "not judged yet" (ReasonPending) while a search runs, and "not
|
||||||
|
// judged" (ReasonIncomplete) after one failed. Never "none found" from a search that did not finish.
|
||||||
|
// - **It runs to completion**, once at a time, with no bound, on a thread of its own at idle priority (Lower):
|
||||||
|
// a fresh one when the last complete one is Every old, when SearchAgain asks (an apply changed what root
|
||||||
|
// controls), or when a filesystem appeared that the last one did not walk. One that fails is tried again
|
||||||
|
// after a back-off that doubles from FailedBackoff up to MaxBackoff.
|
||||||
|
// - **Resumable**: the walk's progress is kept in ProgressPath every CheckpointEvery, and a search started
|
||||||
|
// again — after an engine restart, an update, a reboot or a failure — carries on from it while it is
|
||||||
|
// younger than FreshFor. A search SearchAgain asked for never resumes an older walk: it starts from nothing.
|
||||||
|
// - **Kept** in Path, beside the engine's other state, with when it started and finished: an engine restart
|
||||||
|
// serves the last complete search rather than judging nothing until a new walk ends.
|
||||||
|
//
|
||||||
|
// Why a complete search stays sound for a day and more, and a resumed walk is sound, is said in the rootsearch
|
||||||
|
// package and setuid_walk.go: only root makes a setuid-root file, never the agent account.
|
||||||
|
type SetuidCache struct {
|
||||||
|
// Every and FreshFor are rootsearch's when zero.
|
||||||
|
Every, FreshFor time.Duration
|
||||||
|
// Path is the file the last complete search is kept in, and ProgressPath the walk under way; empty keeps
|
||||||
|
// either in memory only. The engine's own state directory, which only root writes: a file there that is
|
||||||
|
// not a regular file the engine's account owns and alone may write is not read (keptFile).
|
||||||
|
Path, ProgressPath string
|
||||||
|
// Lower is called on the search's own goroutine before it walks (LowerThisThread on a serving engine); a
|
||||||
|
// priority it cannot lower fails the search rather than walk the disks at full priority. Nil lowers nothing.
|
||||||
|
Lower func() error
|
||||||
|
// Say is where a search's end and a kept file refused are said; nil says nothing.
|
||||||
|
Say func(string)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
wg sync.WaitGroup
|
||||||
|
loaded bool
|
||||||
|
last *setuidResult // the last complete search
|
||||||
|
unfinished *SetuidProgress // a walk that did not finish, to resume
|
||||||
|
progress *SearchProgress // how far the running search got, as of its last checkpoint
|
||||||
|
running bool
|
||||||
|
gen int // counts SearchNow: a search started under an older one is thrown away
|
||||||
|
cancel context.CancelFunc // ends the running search's walk
|
||||||
|
startedAt time.Time // when the search running, or the last one, was started (a resumed one: its start)
|
||||||
|
attemptAt time.Time // when the running or last search was started or resumed
|
||||||
|
endedAt time.Time // when the last search ended, on the same clock
|
||||||
|
again bool // asked to search again since the running or last one started
|
||||||
|
failedAt time.Time
|
||||||
|
failed error
|
||||||
|
backoff time.Duration
|
||||||
|
starts int // how many searches were started or resumed, for a test
|
||||||
|
}
|
||||||
|
|
||||||
|
// setuidResult is one complete search, as kept.
|
||||||
|
type setuidResult struct {
|
||||||
|
// Started and Finished are on the caller's clock: Finished is Started plus how long the search took.
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Finished time.Time `json:"finished"`
|
||||||
|
// Mounts are the filesystems it walked; Found the setuid-root programs no package owns.
|
||||||
|
Mounts Paths `json:"mounts"`
|
||||||
|
Found Paths `json:"found"`
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// FailedBackoff is the first wait after a search failed, doubling to MaxBackoff.
|
||||||
|
FailedBackoff = time.Hour
|
||||||
|
MaxBackoff = 6 * time.Hour
|
||||||
|
// AgainAfter is the least time between the starts of two searches that SearchAgain or a new filesystem
|
||||||
|
// asked for: a run of applies does not walk the disks back to back.
|
||||||
|
AgainAfter = time.Hour
|
||||||
|
// clockSkew is how far in the future a kept search may say it started and still judge: a clock set back
|
||||||
|
// further makes it no verdict, never a fresh one.
|
||||||
|
clockSkew = 5 * time.Minute
|
||||||
|
// SetuidSearchFile and SetuidProgressFile are the files' names beside the engine's state: the last
|
||||||
|
// complete search, and the walk under way.
|
||||||
|
SetuidSearchFile = "setuid-search.json"
|
||||||
|
SetuidProgressFile = "setuid-search-progress.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SetuidNotJudged is the search's answer when no complete search judges: none has finished yet, the last one is
|
||||||
|
// older than FreshFor, or every one since failed. The judge says the way is not judged, never that there is none.
|
||||||
|
type SetuidNotJudged struct {
|
||||||
|
// Pending is true while a search runs; false after one failed.
|
||||||
|
Pending bool
|
||||||
|
Since time.Time
|
||||||
|
Err error
|
||||||
|
// Last is when the last complete search started, when there is one too old to judge.
|
||||||
|
Last time.Time
|
||||||
|
// Progress is how far the running search got, as of its last checkpoint: nil when none runs.
|
||||||
|
Progress *SearchProgress
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchProgress is how far a running search got, as of its last checkpoint, for anybody to read (novox/hq
|
||||||
|
// issue 361): counts, times and mount points only. **Never a path the walk visited or found**: the cursor and
|
||||||
|
// the files found are under accounts' homes as often as not, and stay in the engine's state and journal.
|
||||||
|
type SearchProgress struct {
|
||||||
|
// Started is when the search began, on the cache's clock.
|
||||||
|
Started time.Time
|
||||||
|
// Filesystems is how many it walks; Done how many it has walked to their end; Walking the mount point of
|
||||||
|
// the one it walks now, empty when every one is walked.
|
||||||
|
Filesystems, Done int
|
||||||
|
Walking string
|
||||||
|
// Visited is how many entries it has gone past; Found how many setuid- or setgid-root files it found so
|
||||||
|
// far, before the package manager is asked about them.
|
||||||
|
Visited int64
|
||||||
|
Found int
|
||||||
|
// AtLeast says Visited is a lower bound: the walk was begun by an engine that did not count.
|
||||||
|
AtLeast bool
|
||||||
|
// Kept is when its progress was last kept, zero before the first time.
|
||||||
|
Kept time.Time
|
||||||
|
// Finished is every filesystem walked to its end, in order, and how long its walk took.
|
||||||
|
Finished []SearchedFilesystem
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchedFilesystem is one filesystem a search walked to its end. Took is negative when not known: walked by
|
||||||
|
// an engine before it counted.
|
||||||
|
type SearchedFilesystem struct {
|
||||||
|
Mount string
|
||||||
|
Took time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// progressOf is p as anybody may read it. Counts and mount points, no I/O.
|
||||||
|
func progressOf(p *SetuidProgress, kept time.Time) *SearchProgress {
|
||||||
|
sp := &SearchProgress{Started: p.Started, Filesystems: len(p.Mounts), Done: len(p.Done), Visited: p.Visited,
|
||||||
|
Found: len(p.Found), Kept: kept, AtLeast: !p.Counted}
|
||||||
|
for i, m := range p.Done {
|
||||||
|
took := time.Duration(-1)
|
||||||
|
if i < len(p.Took) {
|
||||||
|
took = p.Took[i]
|
||||||
|
}
|
||||||
|
sp.Finished = append(sp.Finished, SearchedFilesystem{Mount: m, Took: took})
|
||||||
|
}
|
||||||
|
for _, m := range p.Mounts {
|
||||||
|
if !slices.Contains(p.Done, m) {
|
||||||
|
sp.Walking = m
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sp
|
||||||
|
}
|
||||||
|
|
||||||
|
// Words are the progress in one line, as the verdict says it beside "not judged yet".
|
||||||
|
func (sp *SearchProgress) Words() string {
|
||||||
|
if sp == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
s := fmt.Sprintf("progress: %d of %d filesystems walked", sp.Done, sp.Filesystems)
|
||||||
|
if sp.Walking != "" {
|
||||||
|
s += ", walking " + sp.Walking
|
||||||
|
}
|
||||||
|
atLeast := ""
|
||||||
|
if sp.AtLeast {
|
||||||
|
atLeast = "at least "
|
||||||
|
}
|
||||||
|
s += fmt.Sprintf(", %s%d entries visited, %d setuid- or setgid-root files found so far", atLeast, sp.Visited, sp.Found)
|
||||||
|
if sp.Kept.IsZero() {
|
||||||
|
s += ", not kept yet"
|
||||||
|
} else {
|
||||||
|
s += ", kept " + sp.Kept.Local().Format("2006-01-02 15:04:05")
|
||||||
|
}
|
||||||
|
var took []string
|
||||||
|
for _, f := range sp.Finished {
|
||||||
|
if f.Took < 0 {
|
||||||
|
took = append(took, f.Mount+" in a time not known")
|
||||||
|
} else {
|
||||||
|
took = append(took, f.Mount+" in "+f.Took.Round(time.Second).String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(took) > 0 {
|
||||||
|
s += "; walked " + strings.Join(took, ", ")
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SetuidNotJudged) Error() string {
|
||||||
|
stale := ""
|
||||||
|
if !e.Last.IsZero() {
|
||||||
|
stale = fmt.Sprintf("; the last complete one, started %s, is too old to judge", e.Last.Local().Format("2006-01-02 15:04"))
|
||||||
|
}
|
||||||
|
if e.Pending {
|
||||||
|
return fmt.Sprintf("%s: the search for setuid programs, started %s, has not finished yet%s", ReasonPending,
|
||||||
|
e.Since.Local().Format("2006-01-02 15:04"), stale)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s: the search for setuid programs did not complete (last tried %s: %v); it is tried "+
|
||||||
|
"again later%s", ReasonIncomplete, e.Since.Local().Format("2006-01-02 15:04"), e.Err, stale)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reasons of a root verdict the setuid search could not answer.
|
||||||
|
const (
|
||||||
|
ReasonPending = rootsearch.ReasonPending
|
||||||
|
ReasonIncomplete = rootsearch.ReasonIncomplete
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *SetuidCache) every() time.Duration {
|
||||||
|
if c.Every > 0 {
|
||||||
|
return c.Every
|
||||||
|
}
|
||||||
|
return rootsearch.Every
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *SetuidCache) freshFor() time.Duration {
|
||||||
|
if c.FreshFor > 0 {
|
||||||
|
return c.FreshFor
|
||||||
|
}
|
||||||
|
return rootsearch.FreshFor
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchAgain asks for a new search at the next look: an apply changed what root controls (an account, a group,
|
||||||
|
// a sudo rule, a package, an action). The last complete search keeps judging until it is too old: only root acts
|
||||||
|
// through an apply, and the agent account cannot make a setuid-root file. Nil c is nothing to ask.
|
||||||
|
func (c *SetuidCache) SearchAgain() {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.again = true
|
||||||
|
// The walk under way, or kept, started before root acted: never carried on, even by an engine that
|
||||||
|
// restarts before the next search starts (the review of 2026-10-10). It keeps no progress from now on.
|
||||||
|
c.unfinished = nil
|
||||||
|
c.forgetProgress()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does).
|
||||||
|
// Nil c runs none.
|
||||||
|
func (c *SetuidCache) Running() bool {
|
||||||
|
if c == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.running
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
||||||
|
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
||||||
|
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
||||||
|
// verdict is "not judged yet". Asked at the controller's terminal alone (`nox node setuid-search <node>`): a
|
||||||
|
// search can never make a machine free wrongly, but asked again and again it would keep a machine unjudged
|
||||||
|
// and its disks busy. by is who asked, for the journal. Nil c has nothing to throw away.
|
||||||
|
func (c *SetuidCache) SearchNow(by string) string {
|
||||||
|
if c == nil {
|
||||||
|
return "this engine keeps no search for setuid programs"
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.loaded = true // nothing kept is read after this: it is thrown away
|
||||||
|
what := "no complete search was kept"
|
||||||
|
if c.last != nil {
|
||||||
|
what = "the complete search started " + c.last.Started.Local().Format("2006-01-02 15:04") + " is thrown away"
|
||||||
|
}
|
||||||
|
if c.running {
|
||||||
|
what += ", and the search running since " + c.startedAt.Local().Format("2006-01-02 15:04") + " is ended"
|
||||||
|
if c.cancel != nil {
|
||||||
|
c.cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.gen++
|
||||||
|
c.last, c.unfinished, c.progress = nil, nil, nil
|
||||||
|
c.failed, c.failedAt, c.backoff, c.again = nil, time.Time{}, 0, false
|
||||||
|
c.forgetProgress()
|
||||||
|
if c.Path != "" {
|
||||||
|
if err := os.Remove(c.Path); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
c.say("the search for setuid programs could not forget the complete search it kept: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
said := "a new search for setuid programs starts at the next look, asked by " + by + ": " + what +
|
||||||
|
"; until it completes, whether an account can become root through one is not judged"
|
||||||
|
c.say(said)
|
||||||
|
return said
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetuidSearch is one search: it carries p on to its end — walking what p has not walked, calling checkpoint
|
||||||
|
// as it goes so its progress can be kept — and answers the setuid-root programs no package owns.
|
||||||
|
type SetuidSearch func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error)
|
||||||
|
|
||||||
|
// Look answers the setuid-root programs no package owns as the last complete search found them, and when it
|
||||||
|
// started — while it is fresh — and starts a search in the background when one is due; it never waits for one.
|
||||||
|
// mounts are the filesystems a new search walks. Nil c searches in place, for a caller that wants to.
|
||||||
|
//
|
||||||
|
// **One clock: the caller's.** A search's time is the `now` it was started at, never the wall clock read when it
|
||||||
|
// finished: due-ness compares a later `now` with it, and mixing the two clocks made a search never due again
|
||||||
|
// whenever the caller's clock ran behind the wall clock (found while checking issue 356). Measured from the
|
||||||
|
// start, a result reads at most one search older than it is, which errs towards not judging.
|
||||||
|
func (c *SetuidCache) Look(now time.Time, mounts []string, search SetuidSearch) ([]string, time.Time, error) {
|
||||||
|
if c == nil {
|
||||||
|
p := &SetuidProgress{Started: now, Mounts: slices.Clone(mounts), Counted: true}
|
||||||
|
found, err := search(context.Background(), p, func() {})
|
||||||
|
return found, now, err
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if !c.loaded {
|
||||||
|
c.loaded = true
|
||||||
|
c.last = c.loadResult()
|
||||||
|
c.unfinished = c.loadProgress()
|
||||||
|
}
|
||||||
|
var age time.Duration
|
||||||
|
if c.last != nil {
|
||||||
|
age = now.Sub(c.last.Started)
|
||||||
|
}
|
||||||
|
// Fresh only while it walked every filesystem a search would walk now: one mounted since is not judged
|
||||||
|
// by a search that never saw it.
|
||||||
|
fresh := c.last != nil && age > -clockSkew && age < c.freshFor() && covers(c.last.Mounts, mounts)
|
||||||
|
// Asked for (an apply, a new filesystem): at most once an AgainAfter, counted from when the last search
|
||||||
|
// ended, so a walk of hours is not followed by another at once, apply after apply.
|
||||||
|
gap := now.Sub(c.attemptAt)
|
||||||
|
if !c.endedAt.IsZero() {
|
||||||
|
gap = now.Sub(c.endedAt)
|
||||||
|
}
|
||||||
|
due := !c.running && (c.failed == nil || now.Sub(c.failedAt) >= c.backoff) &&
|
||||||
|
(!fresh || age >= c.every() || (c.again && (c.attemptAt.IsZero() || gap >= AgainAfter)))
|
||||||
|
if due {
|
||||||
|
c.start(now, mounts, search)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case fresh:
|
||||||
|
return slices.Clone(c.last.Found), c.last.Started, nil
|
||||||
|
case c.running:
|
||||||
|
return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: c.startedAt, Last: c.lastStarted(),
|
||||||
|
Progress: c.progress}
|
||||||
|
case c.failed != nil:
|
||||||
|
return nil, time.Time{}, &SetuidNotJudged{Since: c.failedAt, Err: c.failed, Last: c.lastStarted()}
|
||||||
|
default:
|
||||||
|
// Not fresh, not running, never failed: cannot be, as not fresh is due. Never a verdict.
|
||||||
|
return nil, time.Time{}, &SetuidNotJudged{Pending: true, Since: now, Last: c.lastStarted()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *SetuidCache) lastStarted() time.Time {
|
||||||
|
if c.last == nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return c.last.Started
|
||||||
|
}
|
||||||
|
|
||||||
|
// resumable is the walk that did not finish, when a search may carry it on: younger than FreshFor, and no
|
||||||
|
// SearchAgain since — an apply that moved root's things wants a walk that starts after it. Anything else is
|
||||||
|
// thrown away. c.mu held.
|
||||||
|
func (c *SetuidCache) resumable(now time.Time) *SetuidProgress {
|
||||||
|
p := c.unfinished
|
||||||
|
c.unfinished = nil
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
age := now.Sub(p.Started)
|
||||||
|
if c.again || age <= -clockSkew || age >= c.freshFor() {
|
||||||
|
c.forgetProgress()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// start runs one search in the background, with no bound: a walk that did not finish carried on, or a new one.
|
||||||
|
// c.mu held.
|
||||||
|
func (c *SetuidCache) start(now time.Time, mounts []string, search SetuidSearch) {
|
||||||
|
p := c.resumable(now)
|
||||||
|
resumed := p != nil
|
||||||
|
if p == nil {
|
||||||
|
p = &SetuidProgress{ID: fmt.Sprintf("%x", now.UnixNano()), Started: now, Mounts: slices.Clone(mounts),
|
||||||
|
Cursor: map[string]string{}, Counted: true}
|
||||||
|
}
|
||||||
|
c.running, c.startedAt, c.attemptAt, c.again = true, p.Started, now, false
|
||||||
|
c.starts++
|
||||||
|
// What a look says until the first checkpoint: where the kept walk was, or nothing walked.
|
||||||
|
kept := p.Kept
|
||||||
|
c.progress = progressOf(p, kept)
|
||||||
|
if resumed {
|
||||||
|
c.say(fmt.Sprintf("the search for setuid programs started %s carries on where it was (%d of %d "+
|
||||||
|
"filesystems walked)", p.Started.Local().Format("2006-01-02 15:04"), len(p.Done), len(p.Mounts)))
|
||||||
|
}
|
||||||
|
began := time.Now()
|
||||||
|
gen := c.gen
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
c.cancel = cancel
|
||||||
|
c.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer c.wg.Done()
|
||||||
|
var found []string
|
||||||
|
var err error
|
||||||
|
if c.Lower != nil {
|
||||||
|
// The goroutine stays on its lowered thread until it ends, and the thread ends with it.
|
||||||
|
if lerr := c.Lower(); lerr != nil {
|
||||||
|
err = fmt.Errorf("its priority could not be lowered, so it did not walk: %w", lerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
unkept := false
|
||||||
|
found, err = search(ctx, p, func() {
|
||||||
|
c.mu.Lock()
|
||||||
|
superseded := c.again || c.gen != gen
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !superseded { // root acted since this walk started: its progress is not to be carried on
|
||||||
|
at := time.Now()
|
||||||
|
p.Kept = at
|
||||||
|
kerr := c.keepProgress(p)
|
||||||
|
// Superseded while it wrote (SearchAgain, SearchNow), after they forgot the kept walk: forgotten
|
||||||
|
// again here, so no engine restarted before this walk ends carries it on (the review of
|
||||||
|
// 2026-10-10). Written outside the lock, so a look never waits on the disk.
|
||||||
|
c.mu.Lock()
|
||||||
|
if c.again || c.gen != gen {
|
||||||
|
c.forgetProgress()
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
if kerr != nil {
|
||||||
|
p.Kept = kept // not kept: the file still says the time before
|
||||||
|
if !unkept {
|
||||||
|
unkept = true
|
||||||
|
c.say("the search for setuid programs could not keep how far it got, and starts over " +
|
||||||
|
"if the engine stops: " + kerr.Error())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
kept = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What a look says from now: computed from p here, on the walk's own goroutine, never read
|
||||||
|
// by a look while the walk changes it.
|
||||||
|
sp := progressOf(p, kept)
|
||||||
|
c.mu.Lock()
|
||||||
|
if c.gen == gen {
|
||||||
|
c.progress = sp
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
took := time.Since(began)
|
||||||
|
cancel()
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.running, c.progress, c.cancel = false, nil, nil
|
||||||
|
c.endedAt = now.Add(took)
|
||||||
|
if c.gen != gen {
|
||||||
|
// Thrown away by SearchNow while it ran: neither a result nor a failure, and nothing to carry on. The
|
||||||
|
// next look starts the search asked for.
|
||||||
|
c.endedAt = time.Time{}
|
||||||
|
// A checkpoint that read its generation just before SearchNow may have kept this walk again.
|
||||||
|
c.forgetProgress()
|
||||||
|
c.say(fmt.Sprintf("the search for setuid programs started %s was ended, as asked (%s this time)",
|
||||||
|
p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
// The back-off counts from when it failed.
|
||||||
|
c.failed, c.failedAt = err, c.endedAt
|
||||||
|
c.backoff = min(max(2*c.backoff, FailedBackoff), MaxBackoff)
|
||||||
|
if !c.again { // superseded by an ask to search again: never carried on
|
||||||
|
c.unfinished = p
|
||||||
|
}
|
||||||
|
c.say(fmt.Sprintf("the search for setuid programs started %s did not complete (%s this time): %v",
|
||||||
|
p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r := &setuidResult{Started: p.Started, Finished: now.Add(took), Mounts: p.Mounts, Found: found}
|
||||||
|
c.last, c.failed, c.backoff = r, nil, 0
|
||||||
|
if err := c.keepResult(r); err != nil {
|
||||||
|
c.say("the search for setuid programs could not be kept, and judges from memory until the engine " +
|
||||||
|
"restarts: " + err.Error())
|
||||||
|
}
|
||||||
|
c.forgetProgress()
|
||||||
|
// The paths in the engine's journal alone: the verdict says one in a home by count.
|
||||||
|
named := ""
|
||||||
|
if len(found) > 0 {
|
||||||
|
named = ": " + strings.Join(found, ", ")
|
||||||
|
}
|
||||||
|
c.say(fmt.Sprintf("the search for setuid programs started %s completed (%s this time): %d no package "+
|
||||||
|
"owns%s", p.Started.Local().Format("2006-01-02 15:04"), took.Round(time.Second), len(found), named))
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *SetuidCache) say(s string) {
|
||||||
|
if c.Say != nil {
|
||||||
|
c.Say(s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// covers says every filesystem a search would walk now was walked.
|
||||||
|
func covers(walked, now []string) bool {
|
||||||
|
for _, m := range now {
|
||||||
|
if !slices.Contains(walked, m) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait is for a test: until the search running, if any, has finished.
|
||||||
|
func (c *SetuidCache) Wait() { c.wg.Wait() }
|
||||||
|
|
||||||
|
func (c *SetuidCache) keepResult(r *setuidResult) error { return keepFile(c.Path, r) }
|
||||||
|
|
||||||
|
func (c *SetuidCache) keepProgress(p *SetuidProgress) error { return keepFile(c.ProgressPath, p) }
|
||||||
|
|
||||||
|
// forgetProgress removes the kept walk: finished, or thrown away.
|
||||||
|
func (c *SetuidCache) forgetProgress() {
|
||||||
|
if c.ProgressPath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := os.Remove(c.ProgressPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
c.say("the search for setuid programs could not forget the walk it kept: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepFile writes v to path: a new file beside it, readable by its owner alone, renamed over it — a link at
|
||||||
|
// path is replaced, never followed. Empty path keeps nothing.
|
||||||
|
func keepFile(path string, v any) error {
|
||||||
|
if path == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(f.Name())
|
||||||
|
if _, err := f.Write(raw); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := f.Sync(); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(f.Name(), path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadResult reads the kept complete search, or nil: none kept, or one that cannot be believed, said why.
|
||||||
|
func (c *SetuidCache) loadResult() *setuidResult {
|
||||||
|
var r setuidResult
|
||||||
|
if !c.loadFile(c.Path, "complete search", &r) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if r.Started.IsZero() || r.Finished.Before(r.Started) || len(r.Mounts) == 0 {
|
||||||
|
c.say("the kept search for setuid programs says no complete search, and a new one runs")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &r
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadProgress reads the kept walk, or nil.
|
||||||
|
func (c *SetuidCache) loadProgress() *SetuidProgress {
|
||||||
|
var p SetuidProgress
|
||||||
|
if !c.loadFile(c.ProgressPath, "walk under way", &p) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if p.Started.IsZero() || len(p.Mounts) == 0 {
|
||||||
|
c.say("the kept walk for setuid programs says no walk, and it starts over")
|
||||||
|
c.forgetProgress()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if p.Cursor == nil {
|
||||||
|
p.Cursor = map[string]string{}
|
||||||
|
}
|
||||||
|
return &p
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *SetuidCache) loadFile(path, what string, v any) bool {
|
||||||
|
if path == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
raw, err := keptFile(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
if err = json.Unmarshal(raw, v); err != nil {
|
||||||
|
err = fmt.Errorf("%s is not JSON: %w", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
c.say(fmt.Sprintf("the kept %s for setuid programs is not read, and a new one runs: %v", what, err))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptFile reads a kept file, believing it only as a regular file, not followed through a link, owned by the
|
||||||
|
// engine's own account and writable by it alone, in a directory that is the same: anything else could have been
|
||||||
|
// written by somebody other than root, and a search it says is no search.
|
||||||
|
func keptFile(path string) ([]byte, error) {
|
||||||
|
if err := ownedAlone(filepath.Dir(path), true); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
info, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ownedAloneInfo(path, info, false); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return io.ReadAll(io.LimitReader(f, 64<<20))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ownedAlone(path string, dir bool) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return ownedAloneInfo(path, info, dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ownedAloneInfo(path string, info fs.FileInfo, dir bool) error {
|
||||||
|
if dir && !info.IsDir() || !dir && !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("%s is not a plain %s", path, map[bool]string{true: "directory", false: "file"}[dir])
|
||||||
|
}
|
||||||
|
st, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("%s's owner could not be read", path)
|
||||||
|
}
|
||||||
|
if int(st.Uid) != os.Geteuid() {
|
||||||
|
return fmt.Errorf("%s is owned by uid %d, not the engine's %d", path, st.Uid, os.Geteuid())
|
||||||
|
}
|
||||||
|
if info.Mode().Perm()&0o022 != 0 {
|
||||||
|
return fmt.Errorf("%s may be written by others than its owner (mode %o)", path, info.Mode().Perm())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,447 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/rootsearch"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The search for setuid programs runs to completion and judges from its last complete result while it is fresh
|
||||||
|
// (novox/hq issue 361): never a bound that ends it, never a verdict from a search that did not finish or is too
|
||||||
|
// old, a new search after root acted, and the result kept beside the engine's state, believed only as root's.
|
||||||
|
|
||||||
|
var at = time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
var root = []string{"/"}
|
||||||
|
|
||||||
|
// gate is a search that waits until released; each search started hands its context on started. found and
|
||||||
|
// err are read when it is released.
|
||||||
|
type gate struct {
|
||||||
|
release chan struct{}
|
||||||
|
started chan context.Context
|
||||||
|
found []string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGate() *gate {
|
||||||
|
return &gate{release: make(chan struct{}, 16), started: make(chan context.Context, 16)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gate) search(ctx context.Context, p *SetuidProgress, _ func()) ([]string, error) {
|
||||||
|
g.started <- ctx
|
||||||
|
<-g.release
|
||||||
|
return g.found, g.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gate) finish(c *SetuidCache) { g.release <- struct{}{}; c.Wait() }
|
||||||
|
|
||||||
|
// starts is how many searches c started.
|
||||||
|
func starts(c *SetuidCache) int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.starts
|
||||||
|
}
|
||||||
|
|
||||||
|
func notJudged(t *testing.T, err error, pending bool) *SetuidNotJudged {
|
||||||
|
t.Helper()
|
||||||
|
var nj *SetuidNotJudged
|
||||||
|
if !errors.As(err, &nj) || nj.Pending != pending {
|
||||||
|
t.Fatalf("want not judged (pending %v), got %v", pending, err)
|
||||||
|
}
|
||||||
|
want := ReasonIncomplete
|
||||||
|
if pending {
|
||||||
|
want = ReasonPending
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(err.Error(), want) {
|
||||||
|
t.Fatalf("the words start %q: %v", want, err)
|
||||||
|
}
|
||||||
|
return nj
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheSearchRunsToCompletionWithNoBound(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
if _, _, err := c.Look(at, root, g.search); err == nil {
|
||||||
|
t.Fatal("a look before the first search finished judged")
|
||||||
|
}
|
||||||
|
ctx := <-g.started
|
||||||
|
if _, ok := ctx.Deadline(); ok {
|
||||||
|
t.Fatal("the search was given a deadline: it must run until it has walked everything")
|
||||||
|
}
|
||||||
|
// Hours later, still running: still pending, never killed, never a second walk beside it.
|
||||||
|
_, _, err := c.Look(at.Add(5*time.Hour), root, g.search)
|
||||||
|
notJudged(t, err, true)
|
||||||
|
if starts(c) != 1 || ctx.Err() != nil {
|
||||||
|
t.Fatalf("a running search was started again or stopped: %d searches, %v", starts(c), ctx.Err())
|
||||||
|
}
|
||||||
|
g.found = []string{"/opt/x"}
|
||||||
|
g.finish(c)
|
||||||
|
found, when, err := c.Look(at.Add(5*time.Hour), root, g.search)
|
||||||
|
if err != nil || len(found) != 1 || !when.Equal(at) {
|
||||||
|
t.Fatalf("the complete search judges, from its start: %v %v %v", found, when, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFreshCompleteSearchJudgesUntilItIsDue(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
if _, _, err := c.Look(at.Add(rootsearch.Every-time.Minute), root, g.search); err != nil {
|
||||||
|
t.Fatalf("a complete search within a day: %v", err)
|
||||||
|
}
|
||||||
|
if starts(c) != 1 {
|
||||||
|
t.Fatalf("searched again before it was due: %d", starts(c))
|
||||||
|
}
|
||||||
|
// Due: a new search starts, and the last complete one judges meanwhile.
|
||||||
|
if _, _, err := c.Look(at.Add(rootsearch.Every), root, g.search); err != nil {
|
||||||
|
t.Fatalf("while the daily search runs, the last complete one still judges: %v", err)
|
||||||
|
}
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("the daily search did not start: %d", starts(c))
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStaleSearchIsNotJudgedNeverFree(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
c.Look(at.Add(rootsearch.Every), root, g.search) // the daily one, which does not finish
|
||||||
|
_, _, err := c.Look(at.Add(rootsearch.FreshFor), root, g.search)
|
||||||
|
nj := notJudged(t, err, true)
|
||||||
|
if !nj.Last.Equal(at) || !strings.Contains(err.Error(), "too old to judge") {
|
||||||
|
t.Fatalf("a stale search says when the last complete one started: %v", err)
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
if _, _, err := c.Look(at.Add(rootsearch.FreshFor), root, g.search); err != nil {
|
||||||
|
t.Fatalf("once the new search completed it judges again: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnIncompleteSearchIsNeverFree(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
g.err = errors.New("the search for setuid programs did not complete its walk")
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
_, _, err := c.Look(at.Add(time.Minute), root, g.search)
|
||||||
|
notJudged(t, err, false)
|
||||||
|
if starts(c) != 1 {
|
||||||
|
t.Fatalf("tried again within its back-off: %d", starts(c))
|
||||||
|
}
|
||||||
|
c.Look(at.Add(FailedBackoff+time.Second), root, g.search)
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("not tried again after its back-off: %d", starts(c))
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
// A stale complete search and a failed new one: still not judged, never the stale one.
|
||||||
|
c = &SetuidCache{}
|
||||||
|
g = newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
g.err = errors.New("pacman could not say who owns the setuid programs found")
|
||||||
|
c.Look(at.Add(rootsearch.Every), root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
if _, _, err := c.Look(at.Add(rootsearch.Every+time.Hour), root, g.search); err != nil {
|
||||||
|
t.Fatalf("a failed search leaves the fresh complete one judging: %v", err)
|
||||||
|
}
|
||||||
|
_, _, err = c.Look(at.Add(rootsearch.FreshFor), root, g.search)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a stale search judged after the newer one failed")
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnApplyThatMovedRootSearchesAgain(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
c.SearchAgain()
|
||||||
|
if _, _, err := c.Look(at.Add(time.Minute), root, g.search); err != nil || starts(c) != 1 {
|
||||||
|
t.Fatalf("asked again within AgainAfter of the last search's end: %v, %d searches", err, starts(c))
|
||||||
|
}
|
||||||
|
if _, _, err := c.Look(at.Add(AgainAfter+time.Second), root, g.search); err != nil {
|
||||||
|
t.Fatalf("the last complete search judges while the new one runs: %v", err)
|
||||||
|
}
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("an apply that moved root did not search again: %d", starts(c))
|
||||||
|
}
|
||||||
|
// Asked again while one runs: one more after it, never two at once.
|
||||||
|
c.SearchAgain()
|
||||||
|
c.Look(at.Add(AgainAfter+time.Minute), root, g.search)
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("a second search started beside the running one: %d", starts(c))
|
||||||
|
}
|
||||||
|
g.finish(c) // ends, on the cache's clock, at about AgainAfter+1s
|
||||||
|
c.Look(at.Add(2*AgainAfter), root, g.search)
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatal("a search started again less than AgainAfter after the last one ended")
|
||||||
|
}
|
||||||
|
c.Look(at.Add(2*AgainAfter+2*time.Second), root, g.search)
|
||||||
|
if starts(c) != 3 {
|
||||||
|
t.Fatalf("the ask made while a search ran was dropped: %d", starts(c))
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
var nilCache *SetuidCache
|
||||||
|
nilCache.SearchAgain()
|
||||||
|
}
|
||||||
|
|
||||||
|
// A filesystem the last complete search did not walk is not judged by it: not judged yet, and a search starts.
|
||||||
|
func TestAFilesystemTheSearchDidNotWalkIsNotJudgedByIt(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
_, _, err := c.Look(at.Add(time.Hour), []string{"/", "/srv"}, g.search)
|
||||||
|
notJudged(t, err, true)
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("a new filesystem was not searched: %d", starts(c))
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
if _, _, err := c.Look(at.Add(time.Hour), []string{"/", "/srv"}, g.search); err != nil {
|
||||||
|
t.Fatalf("once walked it judges: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask to search again forgets the walk under way, so an engine restarted before the next search does not
|
||||||
|
// carry on a walk that started before root acted.
|
||||||
|
func TestAnAskToSearchAgainIsNotLostToARestart(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
paths := func() *SetuidCache {
|
||||||
|
return &SetuidCache{Path: filepath.Join(dir, SetuidSearchFile), ProgressPath: filepath.Join(dir, SetuidProgressFile)}
|
||||||
|
}
|
||||||
|
seen := make(chan *SetuidProgress, 1)
|
||||||
|
first := paths()
|
||||||
|
first.Look(at, root, interrupted(seen))
|
||||||
|
<-seen
|
||||||
|
first.SearchAgain() // an apply moved root's things; then the engine restarts
|
||||||
|
restarted := paths()
|
||||||
|
var with SetuidProgress
|
||||||
|
restarted.Look(at.Add(time.Hour), root, func(ctx context.Context, p *SetuidProgress, _ func()) ([]string, error) {
|
||||||
|
with = *p
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
restarted.Wait()
|
||||||
|
if !with.Started.Equal(at.Add(time.Hour)) || len(with.Found) != 0 {
|
||||||
|
t.Fatalf("the walk from before the apply was carried on: %+v", with)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cursor whose name is not UTF-8 is kept as its bytes: JSON strings would change it, and the resumed walk
|
||||||
|
// would skip what it stood before.
|
||||||
|
func TestACursorOfAnyBytesIsKeptExactly(t *testing.T) {
|
||||||
|
p := &SetuidProgress{Started: at, Mounts: Paths{"/"}, Cursor: Cursors{"/": "/home/agent/\x80/x"},
|
||||||
|
Found: Paths{"/opt/\xff"}}
|
||||||
|
var back SetuidProgress
|
||||||
|
fromJSON(t, mustJSON(t, p), &back)
|
||||||
|
if back.Cursor["/"] != "/home/agent/\x80/x" || back.Found[0] != "/opt/\xff" {
|
||||||
|
t.Fatalf("kept as %q, %q", back.Cursor["/"], back.Found[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCompleteSearchIsKeptAndReadAfterARestart(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, SetuidSearchFile)
|
||||||
|
c := &SetuidCache{Path: path}
|
||||||
|
g := newGate()
|
||||||
|
g.found = []string{"/opt/x"}
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil || info.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("kept readable by its owner alone: %v %v", info, err)
|
||||||
|
}
|
||||||
|
restarted := &SetuidCache{Path: path}
|
||||||
|
found, when, err := restarted.Look(at.Add(time.Hour), root, newGate().search)
|
||||||
|
if err != nil || len(found) != 1 || !when.Equal(at) {
|
||||||
|
t.Fatalf("a restarted engine judges from the kept search: %v %v %v", found, when, err)
|
||||||
|
}
|
||||||
|
restarted.Wait()
|
||||||
|
// And not once it is too old.
|
||||||
|
again := &SetuidCache{Path: path}
|
||||||
|
g2 := newGate()
|
||||||
|
if _, _, err := again.Look(at.Add(rootsearch.FreshFor), root, g2.search); err == nil {
|
||||||
|
t.Fatal("a kept search too old to judge judged")
|
||||||
|
}
|
||||||
|
g2.finish(again)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKeptSearchNotRootsAloneIsNotRead(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, SetuidSearchFile)
|
||||||
|
c := &SetuidCache{Path: path}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
for name, spoil := range map[string]func() error{
|
||||||
|
"writable by others": func() error { return os.Chmod(path, 0o666) },
|
||||||
|
"a link": func() error {
|
||||||
|
other := filepath.Join(dir, "elsewhere.json")
|
||||||
|
if err := os.Rename(path, other); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Symlink(other, path)
|
||||||
|
},
|
||||||
|
"its directory writable by others": func() error { return os.Chmod(dir, 0o777) },
|
||||||
|
} {
|
||||||
|
c = &SetuidCache{Path: path}
|
||||||
|
g = newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c) // a fresh, good one kept again
|
||||||
|
os.Chmod(dir, 0o700)
|
||||||
|
if err := spoil(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
r := &SetuidCache{Path: path, Say: func(s string) { said = append(said, s) }}
|
||||||
|
gr := newGate()
|
||||||
|
if _, _, err := r.Look(at.Add(time.Minute), root, gr.search); err == nil {
|
||||||
|
t.Fatalf("%s: a kept search that is not root's alone judged", name)
|
||||||
|
}
|
||||||
|
if len(said) == 0 {
|
||||||
|
t.Fatalf("%s: refusing the kept search said nothing", name)
|
||||||
|
}
|
||||||
|
gr.finish(r)
|
||||||
|
os.Chmod(dir, 0o700)
|
||||||
|
os.Remove(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKeptSearchFromTheFutureIsNotFresh(t *testing.T) {
|
||||||
|
c := &SetuidCache{}
|
||||||
|
g := newGate()
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
g.finish(c)
|
||||||
|
if _, _, err := c.Look(at.Add(-time.Hour), root, g.search); err == nil {
|
||||||
|
t.Fatal("a clock set back an hour read a search as fresh")
|
||||||
|
}
|
||||||
|
g.finish(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheIdleRunnerLowersItsCommand(t *testing.T) {
|
||||||
|
if _, err := exec.LookPath("nice"); err != nil {
|
||||||
|
t.Skip("no nice here")
|
||||||
|
}
|
||||||
|
out, err := Idle(t.Context(), "sh", "-c", "sleep 0.3; nice")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(out) != "19" {
|
||||||
|
t.Fatalf("the search's commands run at nice %q, want 19", strings.TrimSpace(out))
|
||||||
|
}
|
||||||
|
if _, err := Idle(t.Context(), "sh", "-c", "echo gone >&2; exit 1"); err == nil ||
|
||||||
|
!strings.HasPrefix(err.Error(), "sh exited 1: gone") {
|
||||||
|
t.Fatalf("its failure is worded as CLocale's: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustJSON(t *testing.T, v any) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromJSON(t *testing.T, raw []byte, v any) {
|
||||||
|
t.Helper()
|
||||||
|
if err := json.Unmarshal(raw, v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// interrupted is a search that walks to a cursor, keeps it, and never returns: the engine stopped there.
|
||||||
|
func interrupted(seen chan<- *SetuidProgress) SetuidSearch {
|
||||||
|
return func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error) {
|
||||||
|
p.Found = append(p.Found, "/opt/a")
|
||||||
|
p.Cursor["/"] = "/opt/a"
|
||||||
|
checkpoint()
|
||||||
|
seen <- p
|
||||||
|
select {} // the engine stops here
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASearchInterruptedByARestartCarriesOnAndIsDatedByItsStart(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
paths := func() *SetuidCache {
|
||||||
|
return &SetuidCache{Path: filepath.Join(dir, SetuidSearchFile), ProgressPath: filepath.Join(dir, SetuidProgressFile)}
|
||||||
|
}
|
||||||
|
seen := make(chan *SetuidProgress, 1)
|
||||||
|
first := paths()
|
||||||
|
first.Look(at, root, interrupted(seen))
|
||||||
|
<-seen
|
||||||
|
// The engine restarts two hours later.
|
||||||
|
restarted := paths()
|
||||||
|
var resumedWith SetuidProgress
|
||||||
|
done := make(chan struct{})
|
||||||
|
_, _, err := restarted.Look(at.Add(2*time.Hour), root, func(ctx context.Context, p *SetuidProgress, _ func()) ([]string, error) {
|
||||||
|
resumedWith = *p
|
||||||
|
close(done)
|
||||||
|
return p.Found, nil
|
||||||
|
})
|
||||||
|
notJudged(t, err, true)
|
||||||
|
<-done
|
||||||
|
restarted.Wait()
|
||||||
|
if !resumedWith.Started.Equal(at) || resumedWith.Cursor["/"] != "/opt/a" || len(resumedWith.Found) != 1 {
|
||||||
|
t.Fatalf("the restarted search did not carry on the walk: %+v", resumedWith)
|
||||||
|
}
|
||||||
|
found, when, err := restarted.Look(at.Add(2*time.Hour), root, newGate().search)
|
||||||
|
if err != nil || !when.Equal(at) || len(found) != 1 {
|
||||||
|
t.Fatalf("the complete search is dated by its start, not its resumption: %v %v %v", found, when, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, SetuidProgressFile)); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("the finished walk's progress is still kept: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWalkTooOldOrAskedAgainStartsOver(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
after time.Duration
|
||||||
|
again bool
|
||||||
|
}{"older than it may judge": {rootsearch.FreshFor, false}, "an apply asked for a new one": {time.Hour, true}} {
|
||||||
|
dir := t.TempDir()
|
||||||
|
paths := func() *SetuidCache {
|
||||||
|
return &SetuidCache{Path: filepath.Join(dir, SetuidSearchFile), ProgressPath: filepath.Join(dir, SetuidProgressFile)}
|
||||||
|
}
|
||||||
|
seen := make(chan *SetuidProgress, 1)
|
||||||
|
paths().Look(at, root, interrupted(seen))
|
||||||
|
<-seen
|
||||||
|
restarted := paths()
|
||||||
|
if c.again {
|
||||||
|
restarted.SearchAgain()
|
||||||
|
}
|
||||||
|
var with SetuidProgress
|
||||||
|
restarted.Look(at.Add(c.after), root, func(ctx context.Context, p *SetuidProgress, _ func()) ([]string, error) {
|
||||||
|
with = *p
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
restarted.Wait()
|
||||||
|
if !with.Started.Equal(at.Add(c.after)) || len(with.Cursor) != 0 || len(with.Found) != 0 {
|
||||||
|
t.Fatalf("%s: the old walk was carried on: %+v", name, with)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASearchWhosePriorityCannotBeLoweredDoesNotWalk(t *testing.T) {
|
||||||
|
c := &SetuidCache{Lower: func() error { return errors.New("not permitted") }}
|
||||||
|
walked := false
|
||||||
|
c.Look(at, root, func(context.Context, *SetuidProgress, func()) ([]string, error) { walked = true; return nil, nil })
|
||||||
|
c.Wait()
|
||||||
|
_, _, err := c.Look(at.Add(time.Minute), root, newGate().search)
|
||||||
|
notJudged(t, err, false)
|
||||||
|
if walked {
|
||||||
|
t.Fatal("it walked at full priority")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
// The walk for setuid programs yields to a busy disk (novox/hq issue 433). It asks before each directory and each
|
||||||
|
// file it reads, so no burst between two looks is longer than PaceLook, however large a directory. The walk is put in the idle I/O class
|
||||||
|
// (idle_linux.go), but only a disk scheduler that schedules by class honours that (BFQ does). The control-node's
|
||||||
|
// disks run mq-deadline, which does not, and there the walk read at full priority: on 2026-10-11 it kept two
|
||||||
|
// spinning disks saturated for sixteen minutes, the bus whose store shares those disks stalled for half a minute
|
||||||
|
// twice, and the controller lost its lease each time. So the walk also paces itself by the machine's I/O pressure
|
||||||
|
// (PSI), which reads the same whatever the scheduler.
|
||||||
|
//
|
||||||
|
// It looks every PaceLook at the share of the time since its last look in which some task waited on I/O — from
|
||||||
|
// the kernel's cumulative stall counter, not its ten-second average, which moves only every two seconds and would
|
||||||
|
// let the walk read blind that long. Above PaceAbove it pauses, PaceStep at a time, until the share falls below.
|
||||||
|
//
|
||||||
|
// **The walk keeps at least a tenth of the time, whatever the pressure** (the review of #79): a run of pauses is
|
||||||
|
// at most PaceRatio times as long as the walk read since the one before. The control-node's own load sits near
|
||||||
|
// PaceAbove with no walk running, so a walk that paused for as long as the pressure stayed high would take days,
|
||||||
|
// outlive the age a search judges for, and leave the machine unjudged — the fault issue 361 repaired. With the
|
||||||
|
// floor a walk takes at most 1 + PaceRatio times as long as it would unpaced, and its bursts are PaceLook long.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// PaceAbove is the share of the time since the last look, in percent, in which some task waited on I/O, above
|
||||||
|
// which the walk pauses.
|
||||||
|
PaceAbove = 10.0
|
||||||
|
// PaceLook is how long the walk reads between two looks at the pressure: the longest it reads blind.
|
||||||
|
PaceLook = 250 * time.Millisecond
|
||||||
|
// PaceStep is one pause, after which the pressure is looked at again.
|
||||||
|
PaceStep = 500 * time.Millisecond
|
||||||
|
// PaceRatio bounds a run of pauses to this many times the time the walk read since the run before: the walk
|
||||||
|
// keeps at least 1/(1+PaceRatio) of the time.
|
||||||
|
PaceRatio = 9
|
||||||
|
// PaceAtMost is the longest run of pauses, whatever the walk read before it.
|
||||||
|
PaceAtMost = 30 * time.Second
|
||||||
|
// ioPressureFile is where the kernel says the machine's I/O pressure.
|
||||||
|
ioPressureFile = "/proc/pressure/io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pacer is what the walk asks before it reads a directory. A nil pacer never pauses.
|
||||||
|
type pacer struct {
|
||||||
|
// path is the pressure file read, kept so a test can see the engine's own walker reads the machine's.
|
||||||
|
path string
|
||||||
|
// stalled is the cumulative time some task waited on I/O (PSI's "some total").
|
||||||
|
stalled func() (time.Duration, error)
|
||||||
|
sleep func(context.Context, time.Duration) error
|
||||||
|
now func() time.Time
|
||||||
|
say func(string)
|
||||||
|
|
||||||
|
lastLook time.Time // when the pressure was last looked at
|
||||||
|
readSince time.Time // when the walk last began to read, after a run of pauses or at its start
|
||||||
|
prevAt time.Time // the time of the last sample of the counter
|
||||||
|
prevTotal time.Duration // the counter at that sample
|
||||||
|
saidBlind bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// newPacer paces by the machine's own I/O pressure, and says through say what it cannot do.
|
||||||
|
func newPacer(say func(string)) *pacer {
|
||||||
|
return &pacer{
|
||||||
|
path: ioPressureFile,
|
||||||
|
stalled: func() (time.Duration, error) { return readIOStalled(ioPressureFile) },
|
||||||
|
sleep: sleepCtx,
|
||||||
|
now: time.Now,
|
||||||
|
say: say,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// yield pauses while the machine's I/O pressure is above PaceAbove: looking at most every PaceLook, pausing
|
||||||
|
// PaceStep at a time, a run of pauses at most PaceRatio times what the walk read since the last run and never more
|
||||||
|
// than PaceAtMost. It ends early only with ctx, whose error ends the walk as incomplete. Pressure that cannot be
|
||||||
|
// read pauses nothing, and is said once.
|
||||||
|
func (pc *pacer) yield(ctx context.Context) error {
|
||||||
|
if pc == nil || pc.stalled == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
now := pc.now()
|
||||||
|
if pc.readSince.IsZero() {
|
||||||
|
pc.readSince = now
|
||||||
|
}
|
||||||
|
if !pc.lastLook.IsZero() && now.Sub(pc.lastLook) < PaceLook {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pc.lastLook = now
|
||||||
|
share, known, err := pc.sample(now)
|
||||||
|
if err != nil || !known || share <= PaceAbove {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
budget := min(PaceAtMost, PaceRatio*now.Sub(pc.readSince))
|
||||||
|
for paused := time.Duration(0); paused < budget; {
|
||||||
|
step := min(PaceStep, budget-paused)
|
||||||
|
if err := pc.sleep(ctx, step); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
paused += step
|
||||||
|
share, known, err = pc.sample(pc.now())
|
||||||
|
if err != nil || (known && share <= PaceAbove) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resumed := pc.now()
|
||||||
|
pc.readSince, pc.lastLook = resumed, resumed
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sample is the share of the time since the last sample, in percent, in which some task waited on I/O; known is
|
||||||
|
// false for the first sample, which only sets where the next one counts from.
|
||||||
|
func (pc *pacer) sample(now time.Time) (float64, bool, error) {
|
||||||
|
total, err := pc.stalled()
|
||||||
|
if err != nil {
|
||||||
|
if !pc.saidBlind && pc.say != nil {
|
||||||
|
pc.say(fmt.Sprintf("the walk for setuid programs cannot read the machine's I/O pressure (%v), so it does "+
|
||||||
|
"not pace itself: where the disk's scheduler ignores the idle class, it may load the disk at full "+
|
||||||
|
"priority", err))
|
||||||
|
}
|
||||||
|
pc.saidBlind = true
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
prevAt, prevTotal := pc.prevAt, pc.prevTotal
|
||||||
|
pc.prevAt, pc.prevTotal = now, total
|
||||||
|
elapsed := now.Sub(prevAt)
|
||||||
|
if prevAt.IsZero() || elapsed <= 0 || total < prevTotal {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
return 100 * float64(total-prevTotal) / float64(elapsed), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readIOStalled is the "some total" of a PSI file: the time, cumulative since boot, in which some task waited on I/O.
|
||||||
|
func readIOStalled(path string) (time.Duration, error) {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return parseIOStalled(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseIOStalled(b []byte) (time.Duration, error) {
|
||||||
|
sc := bufio.NewScanner(bytes.NewReader(b))
|
||||||
|
for sc.Scan() {
|
||||||
|
fields := strings.Fields(sc.Text())
|
||||||
|
if len(fields) == 0 || fields[0] != "some" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, f := range fields[1:] {
|
||||||
|
if v, ok := strings.CutPrefix(f, "total="); ok {
|
||||||
|
us, err := strconv.ParseUint(v, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("the I/O pressure's total is not a count of microseconds: %q", v)
|
||||||
|
}
|
||||||
|
return time.Duration(us) * time.Microsecond, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("the I/O pressure names no \"some\" line with a total")
|
||||||
|
}
|
||||||
|
|
||||||
|
// sleepCtx waits d, or until ctx ends.
|
||||||
|
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||||
|
t := time.NewTimer(d)
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-t.C:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/rootsearch"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The walk for setuid programs yields to a busy disk by the machine's I/O pressure (novox/hq issue 433), because
|
||||||
|
// the idle I/O class it is put in is ignored by a disk scheduler that does not schedule by class, as the
|
||||||
|
// control-node's mq-deadline does not — and it keeps a tenth of the time whatever the pressure, so it still ends.
|
||||||
|
|
||||||
|
// fakePace is a pacer whose stall counter, sleeps and clock a test controls.
|
||||||
|
type fakePace struct {
|
||||||
|
// shares is the stall share, in percent, of each interval between two reads of the counter, in turn; the last
|
||||||
|
// is said again once they run out. The first read only sets the counter's start.
|
||||||
|
shares []float64
|
||||||
|
i int
|
||||||
|
total time.Duration
|
||||||
|
lastRead time.Time
|
||||||
|
reads int
|
||||||
|
|
||||||
|
clock time.Time
|
||||||
|
// tick moves the clock on at every look at it, as a walk reading directories takes time; zero does not.
|
||||||
|
tick time.Duration
|
||||||
|
slept []time.Duration
|
||||||
|
said []string
|
||||||
|
fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFake(shares ...float64) *fakePace {
|
||||||
|
return &fakePace{shares: shares, clock: time.Unix(1_000_000, 0)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePace) pacer() *pacer {
|
||||||
|
return &pacer{
|
||||||
|
stalled: func() (time.Duration, error) {
|
||||||
|
if f.fail != nil {
|
||||||
|
return 0, f.fail
|
||||||
|
}
|
||||||
|
f.reads++
|
||||||
|
if !f.lastRead.IsZero() {
|
||||||
|
s := f.shares[min(f.i, len(f.shares)-1)]
|
||||||
|
f.i++
|
||||||
|
f.total += time.Duration(float64(f.clock.Sub(f.lastRead)) * s / 100)
|
||||||
|
}
|
||||||
|
f.lastRead = f.clock
|
||||||
|
return f.total, nil
|
||||||
|
},
|
||||||
|
sleep: func(ctx context.Context, d time.Duration) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.slept = append(f.slept, d)
|
||||||
|
f.clock = f.clock.Add(d)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
now: func() time.Time {
|
||||||
|
f.clock = f.clock.Add(f.tick)
|
||||||
|
return f.clock
|
||||||
|
},
|
||||||
|
say: func(s string) { f.said = append(f.said, s) },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePace) paused() time.Duration {
|
||||||
|
var total time.Duration
|
||||||
|
for _, d := range f.slept {
|
||||||
|
total += d
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWalkPausesWhileTheDiskIsUnderPressureAndGoesOnWhenItEases(t *testing.T) {
|
||||||
|
f := newFake(60, 40, 3)
|
||||||
|
pc := f.pacer()
|
||||||
|
if err := pc.yield(t.Context()); err != nil { // the first look sets where the counter counts from
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(300 * time.Millisecond) // the walk reads
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 60 % since the first look: pause; 40 % during the first pause: pause again; 3 %: go on.
|
||||||
|
if want := []time.Duration{PaceStep, PaceStep}; !slices.Equal(f.slept, want) {
|
||||||
|
t.Fatalf("slept %v, want %v", f.slept, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheFirstLookOnlySetsWhereTheNextCountsFrom(t *testing.T) {
|
||||||
|
f := newFake(99)
|
||||||
|
if err := f.pacer().yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.slept) != 0 {
|
||||||
|
t.Fatalf("slept %v on the first look, which has nothing to compare", f.slept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWalkDoesNotPauseOnAQuietDisk(t *testing.T) {
|
||||||
|
f := newFake(0.4)
|
||||||
|
pc := f.pacer()
|
||||||
|
for range 10 {
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(PaceLook)
|
||||||
|
}
|
||||||
|
if len(f.slept) != 0 {
|
||||||
|
t.Fatalf("slept %v on a quiet disk", f.slept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The control-node's own load sits near the bound with no walk running: the walk must still end (issue 361).
|
||||||
|
func TestTheWalkKeepsATenthOfTheTimeWhateverThePressure(t *testing.T) {
|
||||||
|
f := newFake(100)
|
||||||
|
pc := f.pacer()
|
||||||
|
var read time.Duration
|
||||||
|
for range 400 {
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(PaceLook)
|
||||||
|
read += PaceLook
|
||||||
|
}
|
||||||
|
if f.paused() == 0 {
|
||||||
|
t.Fatal("never paused under constant pressure")
|
||||||
|
}
|
||||||
|
if f.paused() > PaceRatio*read {
|
||||||
|
t.Fatalf("paused %v while it read %v: more than %d times as long", f.paused(), read, PaceRatio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the floor promises against the ages a search is judged by: the walk that took 16m31s unpaced on the
|
||||||
|
// control-node on 2026-10-11, slowed to its worst, still completes well inside the time the controller stays quiet
|
||||||
|
// while it waits for a search. A guard on the constants only: that the code keeps to the ratio is proven by
|
||||||
|
// TestTheWalkKeepsATenthOfTheTimeWhateverThePressure.
|
||||||
|
func TestTheSlowestPacedWalkStillJudgesInTime(t *testing.T) {
|
||||||
|
const unpaced = 16*time.Minute + 31*time.Second
|
||||||
|
worst := (1 + PaceRatio) * unpaced
|
||||||
|
if worst >= rootsearch.Quiet {
|
||||||
|
t.Fatalf("a walk paced to its worst takes %v, not less than the %v the controller waits quietly", worst,
|
||||||
|
rootsearch.Quiet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARunOfPausesIsAtMostPaceAtMost(t *testing.T) {
|
||||||
|
f := newFake(100)
|
||||||
|
pc := f.pacer()
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(10 * time.Minute) // read a long time without a look: the ratio would allow 90 minutes
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.paused() != PaceAtMost {
|
||||||
|
t.Fatalf("paused %v in a row, want %v", f.paused(), PaceAtMost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWalkLooksAtThePressureAtMostEveryPaceLook(t *testing.T) {
|
||||||
|
f := newFake(1)
|
||||||
|
pc := f.pacer()
|
||||||
|
for range 5 {
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(PaceLook / 10)
|
||||||
|
}
|
||||||
|
if f.reads != 1 {
|
||||||
|
t.Fatalf("read the pressure %d times within %v, want once", f.reads, PaceLook)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(PaceLook)
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.reads != 2 {
|
||||||
|
t.Fatalf("read the pressure %d times after %v, want twice", f.reads, PaceLook)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pressure that cannot be read pauses nothing, and is said once, with what it means, rather than at every look.
|
||||||
|
func TestAPressureThatCannotBeReadIsSaidOnce(t *testing.T) {
|
||||||
|
f := newFake(0)
|
||||||
|
f.fail = errors.New("no such file")
|
||||||
|
pc := f.pacer()
|
||||||
|
for range 3 {
|
||||||
|
if err := pc.yield(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.clock = f.clock.Add(PaceLook)
|
||||||
|
}
|
||||||
|
if len(f.slept) != 0 || len(f.said) != 1 {
|
||||||
|
t.Fatalf("slept %v, said %q; want no pause and one line", f.slept, f.said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(f.said[0], "full priority") {
|
||||||
|
t.Errorf("the line does not say what it means for the disk: %q", f.said[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The engine's own walk is paced by the machine's pressure, and says through the search's own channel.
|
||||||
|
func TestTheEnginesWalkIsPacedByTheMachinesPressure(t *testing.T) {
|
||||||
|
var said []string
|
||||||
|
w := engineWalker(nil, func(s string) { said = append(said, s) })
|
||||||
|
if w.pace == nil || w.pace.path != ioPressureFile || w.pace.stalled == nil {
|
||||||
|
t.Fatalf("the engine's walker is not paced by %s: %+v", ioPressureFile, w.pace)
|
||||||
|
}
|
||||||
|
w.pace.say("x")
|
||||||
|
if len(said) != 1 {
|
||||||
|
t.Fatal("the engine's walker does not say through the search's channel")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// many is a tree of enough directories that a walk reaches a look at the pressure partway.
|
||||||
|
func many(t *testing.T) string {
|
||||||
|
names := []string{"bin/", "bin/su!", "bin/ls", "sbin/", "sbin/mount!", "usr/lib/helper%"}
|
||||||
|
for _, d := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
|
||||||
|
names = append(names, "usr/share/"+d+"/")
|
||||||
|
}
|
||||||
|
return tree(t, names...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk ended while it pauses is an incomplete walk, never one that found nothing.
|
||||||
|
func TestAWalkEndedWhileItPausesIsIncomplete(t *testing.T) {
|
||||||
|
top := many(t)
|
||||||
|
f := newFake(80)
|
||||||
|
f.tick = 100 * time.Millisecond
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
pc := f.pacer()
|
||||||
|
// The engine stops while the walk waits for the disk: the context ends inside the pause, not before the walk.
|
||||||
|
pc.sleep = func(ctx context.Context, d time.Duration) error {
|
||||||
|
f.slept = append(f.slept, d)
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
w := mine()
|
||||||
|
w.pace = pc
|
||||||
|
err := w.walk(ctx, &SetuidProgress{Mounts: []string{top}}, func() {})
|
||||||
|
if len(f.slept) == 0 {
|
||||||
|
t.Fatal("the walk never paused, so this proves nothing about a walk ended while it pauses")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("a walk ended while it paused answered %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paced, the walk finds what it finds unpaced: pausing changes when, never what.
|
||||||
|
func TestAPacedWalkFindsWhatAnUnpacedWalkFinds(t *testing.T) {
|
||||||
|
top := many(t)
|
||||||
|
plain := walkAll(t, mine(), top)
|
||||||
|
|
||||||
|
f := newFake(80, 2)
|
||||||
|
f.tick = 100 * time.Millisecond
|
||||||
|
w := mine()
|
||||||
|
w.pace = f.pacer()
|
||||||
|
paced := walkAll(t, w, top)
|
||||||
|
if !slices.Equal(plain, paced) {
|
||||||
|
t.Fatalf("paced walk found %v, unpaced %v", paced, plain)
|
||||||
|
}
|
||||||
|
if len(f.slept) == 0 {
|
||||||
|
t.Fatal("the walk read its directories without asking the pacer: it never paused under pressure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheIOPressureIsReadAsTheKernelWritesIt(t *testing.T) {
|
||||||
|
got, err := parseIOStalled([]byte("some avg10=26.24 avg60=20.07 avg300=27.47 total=123456789\n" +
|
||||||
|
"full avg10=21.90 avg60=15.77 avg300=22.47 total=98765432\n"))
|
||||||
|
if err != nil || got != 123456789*time.Microsecond {
|
||||||
|
t.Fatalf("read %v, %v; want 123456789 µs", got, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "full avg10=1.00 avg60=0 avg300=0 total=1\n", "some avg10=1 total=x\n"} {
|
||||||
|
if _, err := parseIOStalled([]byte(bad)); err == nil {
|
||||||
|
t.Errorf("%q was read as a pressure", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One directory of many files is paced as it is read, not only between directories: a file's own details may be
|
||||||
|
// read from the disk too.
|
||||||
|
func TestADirectoryOfManyFilesIsPacedAsItIsRead(t *testing.T) {
|
||||||
|
names := []string{"bin/"}
|
||||||
|
for i := range 20 {
|
||||||
|
names = append(names, "bin/f"+strings.Repeat("x", i+1))
|
||||||
|
}
|
||||||
|
top := tree(t, names...)
|
||||||
|
f := newFake(80, 2)
|
||||||
|
f.tick = 100 * time.Millisecond
|
||||||
|
w := mine()
|
||||||
|
w.pace = f.pacer()
|
||||||
|
walkAll(t, w, top)
|
||||||
|
// Two directories alone would be looked at once, 200 ms apart, within one look: no pause at all.
|
||||||
|
if len(f.slept) == 0 {
|
||||||
|
t.Fatal("a directory of twenty files was read without a pause under pressure")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// How far a running search got is readable through the verdict (novox/hq issue 361): counts, times and mount
|
||||||
|
// points, counted once across a resume, and never a path the walk visited or found.
|
||||||
|
|
||||||
|
// A walk stopped midway and resumed counts every entry once, as the walk that was never stopped does, and says
|
||||||
|
// how long each filesystem took.
|
||||||
|
func TestTheProgressCountsEachEntryOnceAcrossAResume(t *testing.T) {
|
||||||
|
top := tree(t, "a/1!", "a/2", "a-b/3!", "b/c/4!", "b/c/5", "b/d!", "c/6!", "d/7!")
|
||||||
|
other := tree(t, "x/1", "y!")
|
||||||
|
whole := &SetuidProgress{Mounts: []string{top, other}}
|
||||||
|
if err := mine().walk(t.Context(), whole, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// top and the 14 entries below it, and other and its 3.
|
||||||
|
if whole.Visited != 15+4 || len(whole.Took) != 2 || whole.Took[0] < 0 || whole.Walking != 0 {
|
||||||
|
t.Fatalf("the whole walk: visited %d, took %v, walking %v", whole.Visited, whole.Took, whole.Walking)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := mine()
|
||||||
|
w.every = 0
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
p := &SetuidProgress{Mounts: []string{top, other}, Counted: true}
|
||||||
|
var kept []byte
|
||||||
|
err := w.walk(ctx, p, func() {
|
||||||
|
kept = mustJSON(t, p)
|
||||||
|
if strings.HasSuffix(p.Cursor[top], "/b/c/4") {
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("the walk was stopped: %v", err)
|
||||||
|
}
|
||||||
|
var resumed SetuidProgress
|
||||||
|
fromJSON(t, kept, &resumed)
|
||||||
|
if resumed.Visited == 0 || resumed.Visited >= whole.Visited || resumed.Walking < 0 {
|
||||||
|
t.Fatalf("the kept walk counted %d of %d", resumed.Visited, whole.Visited)
|
||||||
|
}
|
||||||
|
if err := mine().walk(t.Context(), &resumed, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resumed.Visited != whole.Visited || len(resumed.Took) != 2 || resumed.Took[0] < 0 {
|
||||||
|
t.Fatalf("the resumed walk counted %d entries (the whole one %d), took %v", resumed.Visited,
|
||||||
|
whole.Visited, resumed.Took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk kept before the times were counted has them not known, never shifted onto the next filesystem, and
|
||||||
|
// its entries visited are a lower bound; the filesystem it was part-way through took a time not known.
|
||||||
|
func TestAWalkKeptWithoutTimesSaysThemNotKnown(t *testing.T) {
|
||||||
|
top, other := tree(t, "a"), tree(t, "b")
|
||||||
|
p := &SetuidProgress{Mounts: []string{top, other}, Done: Paths{top}}
|
||||||
|
if err := mine().walk(t.Context(), p, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(p.Took) != 2 || p.Took[0] >= 0 || p.Took[1] < 0 {
|
||||||
|
t.Fatalf("took %v", p.Took)
|
||||||
|
}
|
||||||
|
if w := progressOf(p, time.Time{}).Words(); !strings.Contains(w, top+" in a time not known") ||
|
||||||
|
!strings.Contains(w, "at least ") {
|
||||||
|
t.Fatalf("the words: %s", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A look while the search runs says how far it got at its last checkpoint, with no path it visited or found.
|
||||||
|
func TestALookSaysHowFarTheRunningSearchGotAndNoPath(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
c := &SetuidCache{ProgressPath: filepath.Join(dir, SetuidProgressFile)}
|
||||||
|
secret := "/home/someone/private/rootshell"
|
||||||
|
checkpointed, release := make(chan struct{}), make(chan struct{})
|
||||||
|
search := func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error) {
|
||||||
|
p.Mounts = Paths{"/", "/home", "/boot"}
|
||||||
|
p.Done, p.Took = Paths{"/"}, []time.Duration{90 * time.Minute}
|
||||||
|
p.Cursor["/home"] = "/home/someone/private"
|
||||||
|
p.Found = Paths{"/usr/bin/odd", secret}
|
||||||
|
p.Visited = 1234567
|
||||||
|
checkpoint()
|
||||||
|
close(checkpointed)
|
||||||
|
<-release
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
_, _, err := c.Look(at, root, search)
|
||||||
|
nj := notJudged(t, err, true)
|
||||||
|
if nj.Progress == nil || nj.Progress.Visited != 0 || nj.Progress.Walking != "/" {
|
||||||
|
t.Fatalf("before the first checkpoint: %+v", nj.Progress)
|
||||||
|
}
|
||||||
|
<-checkpointed
|
||||||
|
_, _, err = c.Look(at.Add(time.Minute), root, search)
|
||||||
|
nj = notJudged(t, err, true)
|
||||||
|
sp := nj.Progress
|
||||||
|
if sp == nil || sp.Filesystems != 3 || sp.Done != 1 || sp.Walking != "/home" || sp.Visited != 1234567 ||
|
||||||
|
sp.Found != 2 || sp.Kept.IsZero() || len(sp.Finished) != 1 || sp.Finished[0].Took != 90*time.Minute {
|
||||||
|
t.Fatalf("the progress: %+v", sp)
|
||||||
|
}
|
||||||
|
words := sp.Words()
|
||||||
|
for _, want := range []string{"1 of 3 filesystems walked", "walking /home", "1234567 entries visited",
|
||||||
|
"2 setuid- or setgid-root files found so far", "kept " + sp.Kept.Local().Format("2006-01-02 15:04:05"),
|
||||||
|
"walked / in 1h30m0s"} {
|
||||||
|
if !strings.Contains(words, want) {
|
||||||
|
t.Errorf("the words lack %q: %s", want, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, leak := range []string{"someone", "private", "odd", "rootshell"} {
|
||||||
|
if strings.Contains(words, leak) || strings.Contains(err.Error(), leak) {
|
||||||
|
t.Errorf("a path leaks (%q): %s / %v", leak, words, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
c.Wait()
|
||||||
|
if _, _, err := c.Look(at.Add(2*time.Minute), root, search); err != nil {
|
||||||
|
t.Fatalf("the complete search judges: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The progress moves every minute; the verdict does not change for it, so the journal says it once.
|
||||||
|
type waiting struct{ progress *SearchProgress }
|
||||||
|
|
||||||
|
func (w *waiting) InDatabase(context.Context, string) ([]string, error) { return nil, nil }
|
||||||
|
func (w *waiting) Session(context.Context, string, []string) (Session, error) {
|
||||||
|
return Session{}, nil
|
||||||
|
}
|
||||||
|
func (w *waiting) Escalation(context.Context, string, []string) ([]string, error) {
|
||||||
|
return nil, &SetuidNotJudged{Pending: true, Since: at, Progress: w.progress}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheProgressMovesWithoutChangingTheVerdict(t *testing.T) {
|
||||||
|
r := &waiting{progress: &SearchProgress{Filesystems: 2, Visited: 10}}
|
||||||
|
j := New(r)
|
||||||
|
j.Set([]Account{{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true}})
|
||||||
|
st, changed := j.Look(t.Context())
|
||||||
|
if !changed || st.Accounts[0].State != Unknown || st.Accounts[0].Search.Visited != 10 {
|
||||||
|
t.Fatalf("first look: changed %v, %+v", changed, st.Accounts[0])
|
||||||
|
}
|
||||||
|
r.progress = &SearchProgress{Filesystems: 2, Visited: 20}
|
||||||
|
st, changed = j.Look(t.Context())
|
||||||
|
if changed || st.Accounts[0].Search.Visited != 20 || strings.Contains(st.Accounts[0].Reason, "progress") {
|
||||||
|
t.Fatalf("second look: changed %v, %+v", changed, st.Accounts[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asked at the terminal, a search starts afresh at the next look: the last complete one is thrown away, kept or
|
||||||
|
// in memory, and a running one is ended and neither judges nor is carried on.
|
||||||
|
func TestSearchNowThrowsAwayTheLastSearchAndStartsAFullOne(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
c := &SetuidCache{Path: filepath.Join(dir, SetuidSearchFile), ProgressPath: filepath.Join(dir, SetuidProgressFile)}
|
||||||
|
g := newGate()
|
||||||
|
g.found = []string{"/home/someone/rootshell"}
|
||||||
|
c.Look(at, root, g.search)
|
||||||
|
<-g.started
|
||||||
|
g.finish(c)
|
||||||
|
if found, _, err := c.Look(at.Add(time.Hour), root, g.search); err != nil || len(found) != 1 {
|
||||||
|
t.Fatalf("the complete search judges: %v %v", found, err)
|
||||||
|
}
|
||||||
|
said := c.SearchNow("jo at the controller's terminal")
|
||||||
|
if !strings.Contains(said, "thrown away") || strings.Contains(said, "someone") {
|
||||||
|
t.Fatalf("said: %s", said)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(c.Path); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("the kept search is still there: %v", err)
|
||||||
|
}
|
||||||
|
// The next look starts a full search at once, though the last ended only an hour ago.
|
||||||
|
ended := make(chan error, 1)
|
||||||
|
running := func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error) {
|
||||||
|
if len(p.Done) != 0 || len(p.Cursor) != 0 {
|
||||||
|
t.Errorf("not a full search: %+v", p)
|
||||||
|
}
|
||||||
|
checkpoint()
|
||||||
|
<-ctx.Done()
|
||||||
|
ended <- ctx.Err()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
_, _, err := c.Look(at.Add(time.Hour+time.Minute), root, running)
|
||||||
|
notJudged(t, err, true)
|
||||||
|
if starts(c) != 2 {
|
||||||
|
t.Fatalf("searches started: %d", starts(c))
|
||||||
|
}
|
||||||
|
// Asked again while it runs: it is ended, judges nothing, fails nothing, and is not carried on.
|
||||||
|
c.SearchNow("jo")
|
||||||
|
if err := <-ended; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("the running search was not ended: %v", err)
|
||||||
|
}
|
||||||
|
c.Wait()
|
||||||
|
if _, err := os.Stat(c.ProgressPath); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("the ended walk is still kept: %v", err)
|
||||||
|
}
|
||||||
|
g2 := newGate()
|
||||||
|
_, _, err = c.Look(at.Add(time.Hour+2*time.Minute), root, g2.search)
|
||||||
|
notJudged(t, err, true)
|
||||||
|
<-g2.started
|
||||||
|
g2.finish(c)
|
||||||
|
if starts(c) != 3 {
|
||||||
|
t.Fatalf("searches started: %d", starts(c))
|
||||||
|
}
|
||||||
|
if _, _, err := c.Look(at.Add(time.Hour+3*time.Minute), root, g2.search); err != nil {
|
||||||
|
t.Fatalf("the new search judges: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFilesystemPartWalkedByAnEngineThatDidNotCountTookATimeNotKnown(t *testing.T) {
|
||||||
|
top := tree(t, "a", "b")
|
||||||
|
p := &SetuidProgress{Mounts: []string{top}, Cursor: Cursors{top: filepath.Join(top, "a")}}
|
||||||
|
if err := mine().walk(t.Context(), p, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(p.Took) != 1 || p.Took[0] >= 0 {
|
||||||
|
t.Fatalf("took %v", p.Took)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A setuid program no package owns counts against an account only where it can execute it, read at every
|
||||||
|
// judgement from the modes on the way (novox/hq issue 361): the operator's copies in a home of mode 0700 are no
|
||||||
|
// way for the agent account, and become one the moment the home is opened, without a new search.
|
||||||
|
|
||||||
|
const homeFind = "/home/op/usr/bin/mount"
|
||||||
|
|
||||||
|
func homeMachine(home FileMode) *agentMachine {
|
||||||
|
m := clean()
|
||||||
|
m.setuid = homeFind + "\n"
|
||||||
|
m.nodes = map[string]FileMode{"/home/op": home}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReachabilityDecidesWhetherAFindCounts(t *testing.T) {
|
||||||
|
op := func(perm fs.FileMode) FileMode { return FileMode{UID: 1500, GID: 1500, Perm: fs.ModeDir | perm} }
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
set func(*agentMachine)
|
||||||
|
counted bool
|
||||||
|
}{
|
||||||
|
{"a home of 0700", func(m *agentMachine) {}, false},
|
||||||
|
{"a home of 0755", func(m *agentMachine) { m.nodes["/home/op"] = op(0o755) }, true},
|
||||||
|
{"a home its group may search, the agent in it", func(m *agentMachine) {
|
||||||
|
m.nodes["/home/op"], m.gids = op(0o710), "1600 1500"
|
||||||
|
}, true},
|
||||||
|
{"a home its group may search, the agent not in it", func(m *agentMachine) { m.nodes["/home/op"] = op(0o710) }, false},
|
||||||
|
{"an ACL on the home", func(m *agentMachine) {
|
||||||
|
m.files["/home/op"] = FileMode{UID: 1500, GID: 1500, Perm: 0o700}
|
||||||
|
m.acls = map[string][]ACLEntry{"/home/op": {{}}}
|
||||||
|
}, true},
|
||||||
|
{"a link on the way", func(m *agentMachine) {
|
||||||
|
m.nodes["/home/op"] = FileMode{UID: 1500, GID: 1500, Perm: fs.ModeSymlink | 0o777}
|
||||||
|
}, true},
|
||||||
|
{"a file the agent may not execute", func(m *agentMachine) {
|
||||||
|
m.nodes["/home/op"] = op(0o755)
|
||||||
|
m.nodes[homeFind] = FileMode{Perm: fs.ModeSetuid | 0o4750}
|
||||||
|
}, false},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
m := homeMachine(FileMode{UID: 1500, GID: 1500, Perm: fs.ModeDir | 0o700})
|
||||||
|
c.set(m)
|
||||||
|
v := lookAgent(t, m)
|
||||||
|
if c.counted != (v.State == Unhealthy) {
|
||||||
|
t.Fatalf("counted %v: %+v", c.counted, v)
|
||||||
|
}
|
||||||
|
if strings.Contains(v.Reason, "/home/op") {
|
||||||
|
t.Fatalf("a path in a home was said: %s", v.Reason)
|
||||||
|
}
|
||||||
|
if c.counted && !strings.Contains(v.Reason, "a setuid-root program no package owns in an account's home, which agent can execute") {
|
||||||
|
t.Fatalf("the reason: %s", v.Reason)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The search keeps every find; whether one counts is read again at every look.
|
||||||
|
func TestAHomeOpenedAfterTheSearchCountsWithoutANewSearch(t *testing.T) {
|
||||||
|
m := homeMachine(FileMode{UID: 1500, GID: 1500, Perm: fs.ModeDir | 0o700})
|
||||||
|
c := &SetuidCache{}
|
||||||
|
j := New(Exec{Run: m.run, Walk: m.walk, Stat: m.stat, ReadFile: m.readFile, ReadDir: m.readDir, ACL: m.acl,
|
||||||
|
Lstat: m.lstat, Cache: c})
|
||||||
|
j.Set([]Account{agent})
|
||||||
|
j.Look(t.Context())
|
||||||
|
c.Wait()
|
||||||
|
if st, _ := j.Look(t.Context()); st.Accounts[0].State != Healthy {
|
||||||
|
t.Fatalf("a find in a closed home: %+v", st.Accounts[0])
|
||||||
|
}
|
||||||
|
m.nodes["/home/op"] = FileMode{UID: 1500, GID: 1500, Perm: fs.ModeDir | 0o755}
|
||||||
|
st, _ := j.Look(t.Context())
|
||||||
|
if st.Accounts[0].State != Unhealthy || starts(c) != 1 {
|
||||||
|
t.Fatalf("the home opened: %+v, %d searches", st.Accounts[0], starts(c))
|
||||||
|
}
|
||||||
|
m.nodes["/home/op"] = FileMode{UID: 1500, GID: 1500, Perm: fs.ModeDir | 0o700}
|
||||||
|
if st, _ := j.Look(t.Context()); st.Accounts[0].State != Healthy || starts(c) != 1 {
|
||||||
|
t.Fatalf("the home closed again: %+v", st.Accounts[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// On the machine's own modes, as their owner (the directories above a test's are its own): a directory without
|
||||||
|
// its owner's search bit hides what is in it.
|
||||||
|
func TestExecutableReadsTheMachinesModes(t *testing.T) {
|
||||||
|
if os.Getuid() == 0 {
|
||||||
|
t.Skip("root searches every directory")
|
||||||
|
}
|
||||||
|
top := tree(t, "home/bin/x!")
|
||||||
|
home := filepath.Join(top, "home")
|
||||||
|
me := os.Getuid()
|
||||||
|
noACL := func(string) ([]ACLEntry, error) { return nil, nil }
|
||||||
|
if err := os.Chmod(home, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(home, 0o700) })
|
||||||
|
if Executable(filepath.Join(home, "bin/x"), me, map[int]bool{}, lstatOf, noACL) {
|
||||||
|
t.Fatal("a file in a directory its account may not search was executable")
|
||||||
|
}
|
||||||
|
if err := os.Chmod(home, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !Executable(filepath.Join(home, "bin/x"), me, map[int]bool{}, lstatOf, noACL) {
|
||||||
|
t.Fatal("a file every directory opens was not executable")
|
||||||
|
}
|
||||||
|
if Executable(filepath.Join(home, "bin/gone"), me, map[int]bool{}, lstatOf, noACL) {
|
||||||
|
t.Fatal("a file gone was executable")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The walk for setuid programs, resumable (novox/hq issue 361).
|
||||||
|
//
|
||||||
|
// The walk is the engine's own, in Go rather than find, so it can say how far it got: filepath.WalkDir visits
|
||||||
|
// every filesystem's tree depth first, each directory's entries in lexical order, so the walk's progress is one
|
||||||
|
// cursor per filesystem — the last path it visited — and everything before the cursor in that order is done.
|
||||||
|
// The progress is kept every CheckpointEvery beside the engine's state (SetuidCache), so an engine restarted,
|
||||||
|
// updated or rebooted midway carries on from its cursor rather than starting over: on the control-node the
|
||||||
|
// whole walk takes far longer than an engine runs between updates.
|
||||||
|
//
|
||||||
|
// **Why a resumed walk is sound.** Only root can make a setuid- or setgid-root file: the agent account can
|
||||||
|
// neither set the bit on root's file nor give its own file to root. A file that appears behind the cursor while
|
||||||
|
// the walk runs, or while it waits to be resumed, was made by root — and root's acts through the mesh start a
|
||||||
|
// new search (apply.MovesRoot), whose walk starts from nothing. A walk is dated by its start, and one resumed
|
||||||
|
// more than FreshFor after its start is thrown away and begun again.
|
||||||
|
|
||||||
|
// SetuidProgress is one walk, as far as it got.
|
||||||
|
type SetuidProgress struct {
|
||||||
|
// ID names the walk; Started is when it began, on the cache's clock: the search is dated by it.
|
||||||
|
ID string `json:"id"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
// Mounts are the filesystems it walks, root first; Done those walked to their end.
|
||||||
|
Mounts Paths `json:"mounts"`
|
||||||
|
Done Paths `json:"done"`
|
||||||
|
// Cursor is, for a filesystem being walked, the last path visited.
|
||||||
|
Cursor Cursors `json:"cursor"`
|
||||||
|
// Found is every setuid- or setgid-root regular file found so far, before the package manager is asked.
|
||||||
|
Found Paths `json:"found"`
|
||||||
|
// Visited is how many entries the walk has gone past: each once, across a resume, as it is kept with the
|
||||||
|
// cursor it matches.
|
||||||
|
Visited int64 `json:"visited"`
|
||||||
|
// Took is how long the walk spent in each of Done, in its order (negative: not known, a walk kept by an
|
||||||
|
// engine before it counted); Walking how long it has spent in the filesystem it walks now. Time the
|
||||||
|
// engine did not run is in neither.
|
||||||
|
Took []time.Duration `json:"took"`
|
||||||
|
Walking time.Duration `json:"walking"`
|
||||||
|
// Kept is when this progress was kept, as the cache wrote it; zero before the first time.
|
||||||
|
Kept time.Time `json:"kept"`
|
||||||
|
// Counted is a walk counted from its start: false for one kept by an engine before it counted, whose
|
||||||
|
// Visited is then a lower bound and whose filesystem under way took a time not known (the review of
|
||||||
|
// 2026-10-10).
|
||||||
|
Counted bool `json:"counted"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paths are paths kept as their bytes. **Never as JSON strings**: encoding/json replaces bytes that are not
|
||||||
|
// UTF-8 with U+FFFD, and a cursor so changed sorts after the names it stood before — a resumed walk would skip
|
||||||
|
// them unread (the review of 2026-10-10). A name of any bytes is any account's to make.
|
||||||
|
type Paths []string
|
||||||
|
|
||||||
|
func (ps Paths) MarshalJSON() ([]byte, error) {
|
||||||
|
raw := make([][]byte, len(ps))
|
||||||
|
for i, p := range ps {
|
||||||
|
raw[i] = []byte(p)
|
||||||
|
}
|
||||||
|
return json.Marshal(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ps *Paths) UnmarshalJSON(b []byte) error {
|
||||||
|
var raw [][]byte
|
||||||
|
if err := json.Unmarshal(b, &raw); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*ps = make(Paths, len(raw))
|
||||||
|
for i, r := range raw {
|
||||||
|
(*ps)[i] = string(r)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cursors are each filesystem's cursor, kept as bytes as Paths are.
|
||||||
|
type Cursors map[string]string
|
||||||
|
|
||||||
|
type keptCursor struct {
|
||||||
|
Mount []byte `json:"mount"`
|
||||||
|
Path []byte `json:"path"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cs Cursors) MarshalJSON() ([]byte, error) {
|
||||||
|
raw := []keptCursor{}
|
||||||
|
for m, p := range cs {
|
||||||
|
raw = append(raw, keptCursor{Mount: []byte(m), Path: []byte(p)})
|
||||||
|
}
|
||||||
|
slices.SortFunc(raw, func(a, b keptCursor) int { return strings.Compare(string(a.Mount), string(b.Mount)) })
|
||||||
|
return json.Marshal(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cs *Cursors) UnmarshalJSON(b []byte) error {
|
||||||
|
var raw []keptCursor
|
||||||
|
if err := json.Unmarshal(b, &raw); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*cs = Cursors{}
|
||||||
|
for _, r := range raw {
|
||||||
|
(*cs)[string(r.Mount)] = string(r.Path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckpointEvery is how often a walk's progress is kept.
|
||||||
|
const CheckpointEvery = time.Minute
|
||||||
|
|
||||||
|
// prunedPaths are never walked: the kernel's own trees, and the container runtimes' layers.
|
||||||
|
var prunedPaths = []string{"/proc", "/sys", "/var/lib/docker", "/var/lib/containers"}
|
||||||
|
|
||||||
|
// walker walks for setuid programs. rootUID and rootGID are the owner and group a match has: root's (0), but
|
||||||
|
// for a test. mountPoints are every filesystem mounted on the machine (MountPoints): a directory that is one is
|
||||||
|
// another filesystem, left without being read — a FUSE mount root cannot read, or a network mount whose
|
||||||
|
// server is away, would fail or hang the walk (the review of 2026-10-10).
|
||||||
|
type walker struct {
|
||||||
|
rootUID, rootGID uint32
|
||||||
|
every time.Duration
|
||||||
|
mountPoints map[string]bool
|
||||||
|
// pace is asked before each directory is read (setuid_pace.go); nil never pauses.
|
||||||
|
pace *pacer
|
||||||
|
}
|
||||||
|
|
||||||
|
// walk carries p on to its end, calling checkpoint every w.every and after each filesystem. It ends early only
|
||||||
|
// with an error — a directory it could not read, or ctx ended — which is an incomplete walk, never "none".
|
||||||
|
// A file or directory that vanished while it walked is not an error: what still exists was walked.
|
||||||
|
func (w walker) walk(ctx context.Context, p *SetuidProgress, checkpoint func()) error {
|
||||||
|
if p.Cursor == nil {
|
||||||
|
p.Cursor = Cursors{}
|
||||||
|
}
|
||||||
|
// One Took for each of Done: a walk kept before they were counted has them not known.
|
||||||
|
for len(p.Took) < len(p.Done) {
|
||||||
|
p.Took = append(p.Took, -1)
|
||||||
|
}
|
||||||
|
p.Took = p.Took[:len(p.Done)]
|
||||||
|
if !p.Counted && len(p.Cursor) > 0 {
|
||||||
|
p.Walking = -1 // walked partly by an engine that did not count its time
|
||||||
|
}
|
||||||
|
last := time.Now()
|
||||||
|
for _, m := range p.Mounts {
|
||||||
|
if slices.Contains(p.Done, m) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.walkMount(ctx, p, m, checkpoint, &last); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.Done = append(p.Done, m)
|
||||||
|
p.Took = append(p.Took, p.Walking)
|
||||||
|
p.Walking = 0
|
||||||
|
delete(p.Cursor, m)
|
||||||
|
checkpoint()
|
||||||
|
last = time.Now()
|
||||||
|
}
|
||||||
|
// Looked at again: a file found that is gone, or no longer setuid-root, is no answer the package manager
|
||||||
|
// can give, and would fail every search that resumes this walk (the review of 2026-10-10).
|
||||||
|
var still Paths
|
||||||
|
for _, f := range p.Found {
|
||||||
|
info, err := os.Lstat(f)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the walk for setuid programs could not read %s again: %w", f, err)
|
||||||
|
}
|
||||||
|
if w.setuidRoot(info) {
|
||||||
|
still = append(still, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(still)
|
||||||
|
p.Found = slices.Compact(still)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w walker) walkMount(ctx context.Context, p *SetuidProgress, mount string, checkpoint func(), last *time.Time) error {
|
||||||
|
top, err := os.Lstat(mount)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the filesystem at %s could not be read: %w", mount, err)
|
||||||
|
}
|
||||||
|
dev, ok := deviceOf(top)
|
||||||
|
if !ok || !top.IsDir() {
|
||||||
|
return fmt.Errorf("the filesystem at %s is not a directory whose device could be read", mount)
|
||||||
|
}
|
||||||
|
var cursor []string
|
||||||
|
if c := p.Cursor[mount]; c != "" {
|
||||||
|
cursor = components(c)
|
||||||
|
}
|
||||||
|
// How long it walks this filesystem, carried on from what a resumed walk had spent in it.
|
||||||
|
began, before := time.Now(), p.Walking
|
||||||
|
spent := func() time.Duration {
|
||||||
|
if before < 0 {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return before + time.Since(began)
|
||||||
|
}
|
||||||
|
defer func() { p.Walking = spent() }()
|
||||||
|
return filepath.WalkDir(mount, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return ctxErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil // vanished while it walked
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the walk for setuid programs could not read %s: %w", path, err)
|
||||||
|
}
|
||||||
|
at := components(path)
|
||||||
|
if cursor != nil && walkOrder(at, cursor) < 0 && !isPrefix(at, cursor) {
|
||||||
|
// Before the cursor and not on its way: visited by the walk this resumes.
|
||||||
|
if d.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case d.IsDir():
|
||||||
|
if path != mount && (slices.Contains(prunedPaths, path) || w.mountPoints[path]) {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the walk for setuid programs could not read %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if d, ok := deviceOf(info); !ok || d != dev {
|
||||||
|
return filepath.SkipDir // another filesystem, walked as its own or not at all (find's -xdev)
|
||||||
|
}
|
||||||
|
// Before its entries are read: a busy disk is waited for here (novox/hq issue 433).
|
||||||
|
if err := w.pace.yield(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case d.Type().IsRegular():
|
||||||
|
// A file's own details may be read from the disk too: a directory of many files is paced as it is read.
|
||||||
|
if err := w.pace.yield(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := d.Info() // the entry's own, never through a link
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the walk for setuid programs could not read %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if w.setuidRoot(info) {
|
||||||
|
p.Found = append(p.Found, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The cursor only moves on: the directories on the way back to it are visited again, and are before it.
|
||||||
|
// Counted here alone, so an entry visited again on the way back to the cursor is counted once.
|
||||||
|
if cursor == nil || walkOrder(at, cursor) > 0 {
|
||||||
|
p.Cursor[mount], cursor = path, at
|
||||||
|
p.Visited++
|
||||||
|
}
|
||||||
|
if time.Since(*last) >= w.every {
|
||||||
|
p.Walking = spent()
|
||||||
|
checkpoint()
|
||||||
|
*last = time.Now()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// setuidRoot is a regular file setuid and owned by root, or setgid to root's group.
|
||||||
|
func (w walker) setuidRoot(info fs.FileInfo) bool {
|
||||||
|
st, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok || !info.Mode().IsRegular() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return uint32(st.Uid) == w.rootUID && info.Mode()&fs.ModeSetuid != 0 ||
|
||||||
|
uint32(st.Gid) == w.rootGID && info.Mode()&fs.ModeSetgid != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func deviceOf(info fs.FileInfo) (uint64, bool) {
|
||||||
|
st, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return uint64(st.Dev), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// components are a path's names, in order; "/" has none.
|
||||||
|
func components(path string) []string {
|
||||||
|
path = strings.Trim(filepath.Clean(path), "/")
|
||||||
|
if path == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return strings.Split(path, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkOrder compares two paths in the order WalkDir visits them: name by name, each in lexical order, a
|
||||||
|
// directory before what is in it.
|
||||||
|
func walkOrder(a, b []string) int {
|
||||||
|
for i := 0; i < len(a) && i < len(b); i++ {
|
||||||
|
if c := strings.Compare(a[i], b[i]); c != 0 {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(a) - len(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPrefix says a is b or a directory b is in.
|
||||||
|
func isPrefix(a, b []string) bool {
|
||||||
|
return len(a) <= len(b) && slices.Equal(a, b[:len(a)])
|
||||||
|
}
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The engine's own walk for setuid programs (novox/hq issue 361): what it finds, what it never follows, and that
|
||||||
|
// a walk interrupted carries on from its cursor.
|
||||||
|
|
||||||
|
// tree makes files under a directory of its own: a name ending in "/" a directory, "!" setuid, "%" setgid.
|
||||||
|
func tree(t *testing.T, names ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
top := t.TempDir()
|
||||||
|
for _, n := range names {
|
||||||
|
mode := os.FileMode(0o755)
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(n, "/"):
|
||||||
|
if err := os.MkdirAll(filepath.Join(top, n), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
case strings.HasSuffix(n, "!"):
|
||||||
|
n, mode = strings.TrimSuffix(n, "!"), mode|os.ModeSetuid
|
||||||
|
case strings.HasSuffix(n, "%"):
|
||||||
|
n, mode = strings.TrimSuffix(n, "%"), mode|os.ModeSetgid
|
||||||
|
}
|
||||||
|
p := filepath.Join(top, n)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(p, mode); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return top
|
||||||
|
}
|
||||||
|
|
||||||
|
// mine is a walker for which this test's own account is root.
|
||||||
|
func mine() walker {
|
||||||
|
return walker{rootUID: uint32(os.Getuid()), rootGID: uint32(os.Getgid()), every: time.Hour}
|
||||||
|
}
|
||||||
|
|
||||||
|
func walkAll(t *testing.T, w walker, top string) []string {
|
||||||
|
t.Helper()
|
||||||
|
p := &SetuidProgress{Mounts: []string{top}}
|
||||||
|
if err := w.walk(t.Context(), p, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return rel(top, p.Found)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rel(top string, paths []string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, p := range paths {
|
||||||
|
out = append(out, strings.TrimPrefix(p, top+"/"))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWalkFindsSetuidAndSetgidRootFilesAndFollowsNoLink(t *testing.T) {
|
||||||
|
top := tree(t, "bin/plain", "bin/rootshell!", "sbin/group%", "deep/a/b/c/x!")
|
||||||
|
if err := os.Symlink(filepath.Join(top, "bin/rootshell"), filepath.Join(top, "link")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Join(top, "bin"), filepath.Join(top, "linkdir")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := walkAll(t, mine(), top)
|
||||||
|
if strings.Join(found, ",") != "bin/rootshell,deep/a/b/c/x,sbin/group" {
|
||||||
|
t.Fatalf("found %v", found)
|
||||||
|
}
|
||||||
|
if got := walkAll(t, walker{rootUID: 4242424, rootGID: 4242424, every: time.Hour}, top); len(got) != 0 {
|
||||||
|
t.Fatalf("files nobody's root owns were found: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWalkThatCannotReadADirectoryIsIncomplete(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root reads every directory")
|
||||||
|
}
|
||||||
|
top := tree(t, "open/x!", "closed/y!")
|
||||||
|
if err := os.Chmod(filepath.Join(top, "closed"), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.Chmod(filepath.Join(top, "closed"), 0o755)
|
||||||
|
p := &SetuidProgress{Mounts: []string{top}}
|
||||||
|
if err := mine().walk(t.Context(), p, func() {}); err == nil {
|
||||||
|
t.Fatal("a walk that could not read a directory was complete")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkOrderIsNameByName(t *testing.T) {
|
||||||
|
// "/a/b" is walked before "/a-c" although '-' sorts before '/': the walk goes name by name.
|
||||||
|
if walkOrder(components("/a/b"), components("/a-c")) >= 0 {
|
||||||
|
t.Fatal("/a/b is walked before /a-c")
|
||||||
|
}
|
||||||
|
if walkOrder(components("/a"), components("/a/b")) >= 0 || !isPrefix(components("/a"), components("/a/b")) {
|
||||||
|
t.Fatal("a directory is walked before what is in it")
|
||||||
|
}
|
||||||
|
if len(components("/")) != 0 {
|
||||||
|
t.Fatal("/ has no names")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk stopped midway carries on from its cursor: what was before the cursor is not walked again, what is after
|
||||||
|
// it is, and every file of the first walk is found.
|
||||||
|
func TestAnInterruptedWalkCarriesOnFromItsCursor(t *testing.T) {
|
||||||
|
top := tree(t, "a/1!", "a/2", "a-b/3!", "b/c/4!", "b/c/5", "b/d!", "c/6!", "d/7!")
|
||||||
|
w := mine()
|
||||||
|
w.every = 0 // a checkpoint at every path
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
p := &SetuidProgress{Mounts: []string{top}}
|
||||||
|
var kept []byte
|
||||||
|
visits := 0
|
||||||
|
err := w.walk(ctx, p, func() {
|
||||||
|
visits++
|
||||||
|
kept = mustJSON(t, p)
|
||||||
|
if strings.HasSuffix(p.Cursor[top], "/b/c/4") {
|
||||||
|
stop() // the engine stops here
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("the walk was stopped: %v", err)
|
||||||
|
}
|
||||||
|
var resumed SetuidProgress
|
||||||
|
fromJSON(t, kept, &resumed)
|
||||||
|
if !strings.HasSuffix(resumed.Cursor[top], "/b/c/4") {
|
||||||
|
t.Fatalf("the kept cursor is %q", resumed.Cursor[top])
|
||||||
|
}
|
||||||
|
// Meanwhile, a file appears before the cursor and one after it.
|
||||||
|
for _, n := range []string{"a/late!", "e/late!"} {
|
||||||
|
tree2(t, top, n)
|
||||||
|
}
|
||||||
|
w.every = time.Hour
|
||||||
|
if err := w.walk(t.Context(), &resumed, func() {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := rel(top, resumed.Found)
|
||||||
|
want := []string{"a-b/3", "a/1", "b/c/4", "b/d", "c/6", "d/7", "e/late"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Fatalf("the resumed walk found %v, want %v: the first walk's finds kept, nothing before the cursor "+
|
||||||
|
"walked again, everything after it walked", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func tree2(t *testing.T, top, n string) {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(top, strings.TrimSuffix(n, "!"))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(p, 0o755|os.ModeSetuid); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A search's commands and walk run on a lowered thread while the rest of the engine keeps its priority.
|
||||||
|
func TestTheSearchRunsOnALoweredThreadOfItsOwn(t *testing.T) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
var err error
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
err = LowerThisThread()
|
||||||
|
}()
|
||||||
|
wg.Wait()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the thread could not be lowered: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Another filesystem's mount point is left without being read, and a file found that is gone, or no longer
|
||||||
|
// setuid-root, when the walk ends is not asked about.
|
||||||
|
func TestTheWalkLeavesMountPointsAndDropsWhatWentMeanwhile(t *testing.T) {
|
||||||
|
top := tree(t, "fuse/secret!", "bin/x!", "bin/y!")
|
||||||
|
w := mine()
|
||||||
|
w.mountPoints = map[string]bool{filepath.Join(top, "fuse"): true}
|
||||||
|
w.every = 0
|
||||||
|
p := &SetuidProgress{Mounts: []string{top}}
|
||||||
|
removed := false
|
||||||
|
if err := w.walk(t.Context(), p, func() {
|
||||||
|
if !removed && strings.HasSuffix(p.Cursor[top], "/bin/y") {
|
||||||
|
removed = true
|
||||||
|
os.Remove(filepath.Join(top, "bin/x"))
|
||||||
|
}
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := rel(top, p.Found); strings.Join(got, ",") != "bin/y" {
|
||||||
|
t.Fatalf("found %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMountPointIsReadAsTheKernelEscapesIt(t *testing.T) {
|
||||||
|
mp := MountPoints("/dev/a /mnt/a\\040b ext4 rw 0 0\n/dev/b /mnt/back\\134slash ext4 rw 0 0\n")
|
||||||
|
if !mp["/mnt/a b"] || !mp[`/mnt/back\slash`] {
|
||||||
|
t.Fatalf("read %v", mp)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,810 @@
|
|||||||
|
package accounts
|
||||||
|
|
||||||
|
// The ways to root the judge looks for beyond uid, groups and sudo (novox/hq ADR 0266, the review of
|
||||||
|
// 2026-10-08), and the ones it does not. Judged and NotJudged are the one written list: the record quotes it,
|
||||||
|
// and a way added here is a line added there.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Judged is every way to root the judge looks for, in the words its reasons use.
|
||||||
|
var Judged = []string{
|
||||||
|
"its uid is 0",
|
||||||
|
"membership of a group that grants root by membership: " + strings.Join(RootGroups, ", "),
|
||||||
|
"any sudo rule for it or one of its groups, as `sudo -l -U` lists it (with or without a password)",
|
||||||
|
"any doas rule permitting it or one of its groups (/etc/doas.conf, /etc/opendoas.conf)",
|
||||||
|
"any polkit rule naming it or one of its groups (/etc/polkit-1/rules.d, /usr/share/polkit-1/rules.d, " +
|
||||||
|
"/etc/polkit-1/localauthority), which is how pkexec and systemd's own actions are granted",
|
||||||
|
"a polkit rule that grants every account: a rule of a .rules file that answers polkit.Result.YES and names " +
|
||||||
|
"no user and no group, or a .pkla section whose Identity is unix-user:* or unix-group:* with a Result of " +
|
||||||
|
"yes — each rule and section judged on its own, comments taken out",
|
||||||
|
"write access to a container runtime's socket (docker, podman, containerd), by owner, group, other or " +
|
||||||
|
"POSIX ACL",
|
||||||
|
"a container runtime's API listening on TCP port 2375 or 2376 (/proc/net/tcp, /proc/net/tcp6), which any " +
|
||||||
|
"local account reaches",
|
||||||
|
"read access to a secret the mesh placed for another account, by owner, group, other or POSIX ACL",
|
||||||
|
"a program that no installed package owns and is setuid with owner root, or setgid with group root, " +
|
||||||
|
"whoever owns it, on every local filesystem mounted without nosuid (container and image layers, " +
|
||||||
|
"network and pseudo filesystems excepted) — searched in the background at most hourly, its last result " +
|
||||||
|
"served with when it was found; until a search has finished, the verdict says not judged",
|
||||||
|
"hard links or symbolic links to other accounts' files left unprotected by the kernel " +
|
||||||
|
"(fs.protected_hardlinks or fs.protected_symlinks is 0)",
|
||||||
|
}
|
||||||
|
|
||||||
|
// NotJudged is what the judge does not look for: each is a way to root it would miss.
|
||||||
|
var NotJudged = []string{
|
||||||
|
"a root-run unit, timer, cron entry or script the account can write",
|
||||||
|
"a directory on root's PATH, or in /etc/profile.d, the account can write",
|
||||||
|
"root's or the operator's ssh keys or authorized_keys readable or writable by it",
|
||||||
|
"a file root writes or reads, in a directory the account owns, other than below its home (the node-engine " +
|
||||||
|
"refuses links there)",
|
||||||
|
"file capabilities (setcap) on a program",
|
||||||
|
"a setuid program a package installed that has a flaw of its own",
|
||||||
|
"a terminal the account shares with a root process (TIOCSTI, sudo without use_pty)",
|
||||||
|
"a polkit rule that grants every account by logic the text does not show (a JavaScript condition " +
|
||||||
|
"other than a named user or group), or every account with an active local session",
|
||||||
|
"a container runtime's API on a TCP port other than 2375 and 2376, or on a unix socket not named here",
|
||||||
|
"a setuid program on a filesystem not judged: a container or image layer (overlay, squashfs), a network " +
|
||||||
|
"or FUSE filesystem",
|
||||||
|
}
|
||||||
|
|
||||||
|
// RuntimeSockets are the container runtimes' sockets: write access to one runs a container as root.
|
||||||
|
var RuntimeSockets = []string{"/run/docker.sock", "/var/run/docker.sock", "/run/podman/podman.sock",
|
||||||
|
"/run/containerd/containerd.sock"}
|
||||||
|
|
||||||
|
// DoasConfigs and PolkitDirs are where those grants live.
|
||||||
|
var (
|
||||||
|
DoasConfigs = []string{"/etc/doas.conf", "/etc/opendoas.conf"}
|
||||||
|
PolkitDirs = []string{"/etc/polkit-1/rules.d", "/usr/share/polkit-1/rules.d", "/etc/polkit-1/localauthority"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// ACLEntry is one named entry of a POSIX ACL: a user's or a group's, and whether it grants read and write
|
||||||
|
// once the ACL's mask is applied.
|
||||||
|
type ACLEntry struct {
|
||||||
|
User bool
|
||||||
|
ID int
|
||||||
|
Read, Write bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tags and bits of the kernel's ACL xattr.
|
||||||
|
const (
|
||||||
|
aclUser = 0x02
|
||||||
|
aclGroup = 0x08
|
||||||
|
aclMask = 0x10
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseACL reads a system.posix_acl_access value: a version, then entries of tag, permission and id; the
|
||||||
|
// named users' and groups' entries are answered with the mask applied.
|
||||||
|
func ParseACL(raw []byte) ([]ACLEntry, error) {
|
||||||
|
if len(raw) < 4 || (len(raw)-4)%8 != 0 {
|
||||||
|
return nil, fmt.Errorf("an ACL of %d bytes is not one", len(raw))
|
||||||
|
}
|
||||||
|
type entry struct {
|
||||||
|
tag, perm uint16
|
||||||
|
id uint32
|
||||||
|
}
|
||||||
|
var es []entry
|
||||||
|
mask := uint16(7)
|
||||||
|
for at := 4; at < len(raw); at += 8 {
|
||||||
|
e := entry{binary.LittleEndian.Uint16(raw[at:]), binary.LittleEndian.Uint16(raw[at+2:]),
|
||||||
|
binary.LittleEndian.Uint32(raw[at+4:])}
|
||||||
|
if e.tag == aclMask {
|
||||||
|
mask = e.perm
|
||||||
|
}
|
||||||
|
es = append(es, e)
|
||||||
|
}
|
||||||
|
var out []ACLEntry
|
||||||
|
for _, e := range es {
|
||||||
|
if e.tag != aclUser && e.tag != aclGroup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p := e.perm & mask
|
||||||
|
out = append(out, ACLEntry{User: e.tag == aclUser, ID: int(e.id), Read: p&4 != 0, Write: p&2 != 0})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// aclOf is a file's ACL from the machine: none when it has none.
|
||||||
|
func aclOf(path string) ([]ACLEntry, error) {
|
||||||
|
buf := make([]byte, 1024)
|
||||||
|
n, err := unix.Getxattr(path, "system.posix_acl_access", buf)
|
||||||
|
if errors.Is(err, unix.ENODATA) || errors.Is(err, unix.EOPNOTSUPP) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, &os.PathError{Op: "getxattr", Path: path, Err: err}
|
||||||
|
}
|
||||||
|
return ParseACL(buf[:n])
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantsByACL says whether an ACL entry gives the account read (or write).
|
||||||
|
func grantsByACL(acl []ACLEntry, uid int, gids map[int]bool, write bool) bool {
|
||||||
|
for _, e := range acl {
|
||||||
|
if (e.User && e.ID == uid) || (!e.User && gids[e.ID]) {
|
||||||
|
if (write && e.Write) || (!write && e.Read) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writable is Readable's twin for the write bits.
|
||||||
|
func Writable(m FileMode, uid int, gids map[int]bool) bool {
|
||||||
|
switch {
|
||||||
|
case uid == 0:
|
||||||
|
return true
|
||||||
|
case m.UID == uid:
|
||||||
|
return m.Perm&0o200 != 0
|
||||||
|
case gids[m.GID]:
|
||||||
|
return m.Perm&0o020 != 0
|
||||||
|
default:
|
||||||
|
return m.Perm&0o002 != 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoasRules is every line of a doas configuration that permits the account or one of its groups.
|
||||||
|
func DoasRules(conf string, account string, groups []string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, line := range strings.Split(conf, "\n") {
|
||||||
|
if i := strings.IndexByte(line, '#'); i >= 0 {
|
||||||
|
line = line[:i]
|
||||||
|
}
|
||||||
|
f := strings.Fields(line)
|
||||||
|
if len(f) < 2 || f[0] != "permit" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
i := 1
|
||||||
|
for i < len(f) && (f[i] == "nopass" || f[i] == "persist" || f[i] == "keepenv" || f[i] == "nolog" ||
|
||||||
|
f[i] == "setenv" || strings.HasPrefix(f[i], "{")) {
|
||||||
|
if strings.HasPrefix(f[i], "{") {
|
||||||
|
for i < len(f) && !strings.HasSuffix(f[i], "}") {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
if i >= len(f) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
who := f[i]
|
||||||
|
if who == account || (strings.HasPrefix(who, ":") && contains(groups, who[1:])) {
|
||||||
|
out = append(out, strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// PolkitNames says whether a polkit rule or authority file names the account or one of its groups.
|
||||||
|
func PolkitNames(text, account string, groups []string) bool {
|
||||||
|
needles := []string{"unix-user:" + account, `"` + account + `"`, `'` + account + `'`}
|
||||||
|
for _, g := range groups {
|
||||||
|
needles = append(needles, "unix-group:"+g, `isInGroup("`+g+`")`, `isInGroup('`+g+`')`)
|
||||||
|
}
|
||||||
|
for _, n := range needles {
|
||||||
|
if strings.Contains(text, n) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// PolkitGrantsEveryone says whether a polkit rule or authority file grants every account, naming none: a .rules
|
||||||
|
// file one of whose rules answers polkit.Result.YES with no condition on a user or a group, or a .pkla one of
|
||||||
|
// whose sections has an Identity of every user or every group with a Result of yes.
|
||||||
|
//
|
||||||
|
// **Each rule is judged on its own, comments taken out first** (the third review of 2026-10-08): a rule naming a
|
||||||
|
// user elsewhere in the file, or a user named only in a comment, must not hide another rule's unconditional
|
||||||
|
// yes. A rule's text is read, not run: a condition the text does not show is listed under NotJudged.
|
||||||
|
func PolkitGrantsEveryone(path, text string) bool {
|
||||||
|
if strings.HasSuffix(path, ".pkla") {
|
||||||
|
var kept []string
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
if t := strings.TrimSpace(line); strings.HasPrefix(t, "#") || strings.HasPrefix(t, ";") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kept = append(kept, line)
|
||||||
|
}
|
||||||
|
for _, section := range strings.Split(strings.Join(kept, "\n"), "[") {
|
||||||
|
var every, yes bool
|
||||||
|
for _, line := range strings.Split(section, "\n") {
|
||||||
|
k, v, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v = strings.TrimSpace(v)
|
||||||
|
switch strings.TrimSpace(k) {
|
||||||
|
case "Identity":
|
||||||
|
for _, id := range strings.Split(v, ";") {
|
||||||
|
if id = strings.TrimSpace(id); id == "unix-user:*" || id == "unix-group:*" {
|
||||||
|
every = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "ResultAny", "ResultActive", "ResultInactive":
|
||||||
|
if v == "yes" {
|
||||||
|
yes = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if every && yes {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, rule := range polkitRules(stripJSComments(text)) {
|
||||||
|
if !strings.Contains(rule, "polkit.Result.YES") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A condition on a user or a group names whom it grants; one on an active local session grants only an
|
||||||
|
// account with a seat, which an agent started through sudo has not (listed under NotJudged).
|
||||||
|
conditioned := false
|
||||||
|
for _, condition := range []string{".user", "isInGroup", "unix-user:", "unix-group:", ".active", ".local"} {
|
||||||
|
if strings.Contains(rule, condition) {
|
||||||
|
conditioned = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !conditioned {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// polkitRules is each rule of a .rules file: the text from one polkit.addRule to the next. Text before the first
|
||||||
|
// is no rule.
|
||||||
|
func polkitRules(text string) []string {
|
||||||
|
parts := strings.Split(text, "addRule(")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return parts[1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripJSComments takes /* … */ and // … comments out of a rule file, leaving what is in strings alone.
|
||||||
|
func stripJSComments(text string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
var quote byte
|
||||||
|
for i := 0; i < len(text); i++ {
|
||||||
|
c := text[i]
|
||||||
|
switch {
|
||||||
|
case quote != 0:
|
||||||
|
b.WriteByte(c)
|
||||||
|
if c == '\\' && i+1 < len(text) {
|
||||||
|
i++
|
||||||
|
b.WriteByte(text[i])
|
||||||
|
} else if c == quote {
|
||||||
|
quote = 0
|
||||||
|
}
|
||||||
|
case c == '"' || c == '\'' || c == '`':
|
||||||
|
quote = c
|
||||||
|
b.WriteByte(c)
|
||||||
|
case c == '/' && i+1 < len(text) && text[i+1] == '*':
|
||||||
|
end := strings.Index(text[i+2:], "*/")
|
||||||
|
if end < 0 {
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
i += end + 3
|
||||||
|
case c == '/' && i+1 < len(text) && text[i+1] == '/':
|
||||||
|
end := strings.IndexByte(text[i:], '\n')
|
||||||
|
if end < 0 {
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
i += end - 1
|
||||||
|
default:
|
||||||
|
b.WriteByte(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RuntimeAPIPorts are the ports a container runtime's API listens on by convention: 2375 plain, 2376 TLS.
|
||||||
|
var RuntimeAPIPorts = []int{2375, 2376}
|
||||||
|
|
||||||
|
// ListeningPorts is every local TCP port in the listening state, from /proc/net/tcp's or tcp6's text.
|
||||||
|
func ListeningPorts(text string) []int {
|
||||||
|
var out []int
|
||||||
|
for i, line := range strings.Split(text, "\n") {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
if i == 0 || len(f) < 4 || f[3] != "0A" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, port, ok := strings.Cut(f[1], ":")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var n int
|
||||||
|
if _, err := fmt.Sscanf(port, "%X", &n); err == nil {
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// pseudoFS are filesystem types no setuid program is installed on, or that are a container's or an image's
|
||||||
|
// own layers: not searched.
|
||||||
|
var pseudoFS = map[string]bool{"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true,
|
||||||
|
"cgroup": true, "cgroup2": true, "securityfs": true, "pstore": true, "bpf": true, "debugfs": true,
|
||||||
|
"tracefs": true, "mqueue": true, "hugetlbfs": true, "configfs": true, "fusectl": true, "autofs": true,
|
||||||
|
"binfmt_misc": true, "efivarfs": true, "overlay": true, "squashfs": true, "nsfs": true, "ramfs": true,
|
||||||
|
"nfs": true, "nfs4": true, "cifs": true, "smb3": true, "9p": true, "virtiofs": true}
|
||||||
|
|
||||||
|
// SuidMounts is every local filesystem a setuid program could run from: the mount points of /proc/mounts whose
|
||||||
|
// type is not pseudo, network or a container's layer, and that are not mounted nosuid. Root first.
|
||||||
|
func SuidMounts(text string) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
if len(f) < 4 || pseudoFS[f[2]] || strings.HasPrefix(f[2], "fuse") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := unescapeMount(f[1])
|
||||||
|
if strings.HasPrefix(at, "/var/lib/docker") || strings.HasPrefix(at, "/var/lib/containers") ||
|
||||||
|
strings.HasPrefix(at, "/proc") || strings.HasPrefix(at, "/sys") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if contains(strings.Split(f[3], ","), "nosuid") || seen[at] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[at] = true
|
||||||
|
out = append(out, at)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i] == "/" || (out[j] != "/" && out[i] < out[j]) })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// MountPoints is every mount point /proc/mounts lists, of any type: the walk leaves each it meets inside
|
||||||
|
// another filesystem without reading it.
|
||||||
|
func MountPoints(text string) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
if f := strings.Fields(line); len(f) >= 2 {
|
||||||
|
out[unescapeMount(f[1])] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unescapeMount is a mount point as /proc/mounts escapes it — a space, a tab, a newline and a backslash as
|
||||||
|
// three octal digits after a backslash — made a path again.
|
||||||
|
func unescapeMount(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for i := 0; i < len(s); i++ {
|
||||||
|
if s[i] == '\\' && i+3 < len(s) && isOctal(s[i+1:i+4]) {
|
||||||
|
b.WriteByte((s[i+1]-'0')<<6 | (s[i+2]-'0')<<3 | (s[i+3] - '0'))
|
||||||
|
i += 3
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b.WriteByte(s[i])
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func isOctal(s string) bool {
|
||||||
|
if len(s) != 3 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if s[i] < '0' || s[i] > '7' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(xs []string, s string) bool {
|
||||||
|
for _, x := range xs {
|
||||||
|
if x == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// setuidSearch is every setuid- or setgid-root regular file on the filesystems of p that no installed package
|
||||||
|
// owns: walked by the engine (setuid_walk.go, resumable from p), then asked of the package manager through the
|
||||||
|
// search's runner (Exec.SearchRun). **No bound of its own** (novox/hq issue 361): it runs until it has walked
|
||||||
|
// everything. A walk that could not complete — a directory it could not read — is an unanswered question, never
|
||||||
|
// "none"; a file that vanished while it walked is not.
|
||||||
|
func (e Exec) setuidSearch(ctx context.Context, p *SetuidProgress, checkpoint func(), mountPoints map[string]bool) ([]string, error) {
|
||||||
|
walk := e.Walk
|
||||||
|
if walk == nil {
|
||||||
|
var say func(string)
|
||||||
|
if e.Cache != nil {
|
||||||
|
say = e.Cache.Say
|
||||||
|
}
|
||||||
|
walk = engineWalker(mountPoints, say).walk
|
||||||
|
}
|
||||||
|
if err := walk(ctx, p, checkpoint); err != nil {
|
||||||
|
return nil, fmt.Errorf("the walk for setuid programs did not complete: %w", err)
|
||||||
|
}
|
||||||
|
if e.SearchRun != nil {
|
||||||
|
e.Run = e.SearchRun
|
||||||
|
}
|
||||||
|
found := slices.Clone(p.Found)
|
||||||
|
slices.Sort(found)
|
||||||
|
return e.unpackaged(ctx, slices.Compact(found))
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineWalker is the walk the engine runs: checkpointed every CheckpointEvery, leaving other filesystems unread,
|
||||||
|
// and paced by the machine's I/O pressure (setuid_pace.go, novox/hq issue 433), saying through say what it cannot do.
|
||||||
|
func engineWalker(mountPoints map[string]bool, say func(string)) walker {
|
||||||
|
return walker{every: CheckpointEvery, mountPoints: mountPoints, pace: newPacer(say)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// errorLines are the lines a command said on its error stream, as the runner words a failure: "<name> exited N:
|
||||||
|
// <its words>", the prefix taken off the first line. Empty lines are dropped.
|
||||||
|
func errorLines(err error, name string) []string {
|
||||||
|
words := err.Error()
|
||||||
|
if strings.HasPrefix(words, name+" exited") {
|
||||||
|
if i := strings.Index(words, ": "); i >= 0 {
|
||||||
|
words = words[i+2:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, l := range strings.Split(words, "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
|
out = append(out, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unpackaged is every path no installed package owns: pacman asked about all of them in one call, else apk
|
||||||
|
// about each. Never a guess: a path neither said owned nor said unowned is an unanswered question.
|
||||||
|
func (e Exec) unpackaged(ctx context.Context, paths []string) ([]string, error) {
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
out, err := e.Run(ctx, "pacman", append([]string{"-Qo"}, paths...)...)
|
||||||
|
if err == nil || !errors.Is(err, exec.ErrNotFound) && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
owned := map[string]bool{}
|
||||||
|
for _, l := range strings.Split(out, "\n") {
|
||||||
|
if p, _, ok := strings.Cut(strings.TrimSpace(l), " is owned by "); ok {
|
||||||
|
owned[p] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unowned := map[string]bool{}
|
||||||
|
if err != nil {
|
||||||
|
// pacman exits 1 when a path is owned by nothing, and says which on its error stream.
|
||||||
|
for _, l := range errorLines(err, "pacman") {
|
||||||
|
if _, p, ok := strings.Cut(l, "No package owns "); ok {
|
||||||
|
unowned[strings.TrimSpace(p)] = true
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("pacman could not say who owns the setuid programs found: %s", l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, p := range paths {
|
||||||
|
switch {
|
||||||
|
case owned[p]:
|
||||||
|
case unowned[p]:
|
||||||
|
out = append(out, p)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("pacman said nothing of who owns %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
var none []string
|
||||||
|
for _, p := range paths {
|
||||||
|
said, err := e.Run(ctx, "apk", "info", "-W", p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("no package manager could say who owns %s: %w", p, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, " is owned by ") {
|
||||||
|
none = append(none, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return none, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moreWays is every way beyond uid, groups and sudo; an unanswered question is an error, never none.
|
||||||
|
func (e Exec) moreWays(ctx context.Context, account string, uid int, groups []string, gids map[int]bool,
|
||||||
|
secrets []string) ([]string, error) {
|
||||||
|
read := e.ReadFile
|
||||||
|
if read == nil {
|
||||||
|
read = os.ReadFile
|
||||||
|
}
|
||||||
|
readDir := e.ReadDir
|
||||||
|
if readDir == nil {
|
||||||
|
readDir = os.ReadDir
|
||||||
|
}
|
||||||
|
acl := e.ACL
|
||||||
|
if acl == nil {
|
||||||
|
acl = aclOf
|
||||||
|
}
|
||||||
|
stat := e.Stat
|
||||||
|
if stat == nil {
|
||||||
|
stat = statOf
|
||||||
|
}
|
||||||
|
var ways []string
|
||||||
|
|
||||||
|
// doas.
|
||||||
|
for _, conf := range DoasConfigs {
|
||||||
|
raw, err := read(conf)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("doas's rules in %s could not be read: %w", conf, err)
|
||||||
|
}
|
||||||
|
for _, r := range DoasRules(string(raw), account, groups) {
|
||||||
|
ways = append(ways, "doas permits it: "+r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// polkit.
|
||||||
|
for _, dir := range PolkitDirs {
|
||||||
|
var named, everyone []string
|
||||||
|
err := walkFiles(readDir, dir, func(path string) error {
|
||||||
|
raw, err := read(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if PolkitNames(string(raw), account, groups) {
|
||||||
|
named = append(named, path)
|
||||||
|
} else if PolkitGrantsEveryone(path, string(raw)) {
|
||||||
|
everyone = append(everyone, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("polkit's rules in %s could not be read: %w", dir, err)
|
||||||
|
}
|
||||||
|
for _, p := range named {
|
||||||
|
ways = append(ways, "a polkit rule names it or a group of it: "+p)
|
||||||
|
}
|
||||||
|
for _, p := range everyone {
|
||||||
|
ways = append(ways, "a polkit rule grants every account: "+p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The container runtimes' sockets.
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, s := range RuntimeSockets {
|
||||||
|
// Two names of one socket are one socket. A test that gives its own files names them as they are.
|
||||||
|
real, err := filepath.EvalSymlinks(s)
|
||||||
|
if e.Stat != nil {
|
||||||
|
real, err = s, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the socket %s could not be read: %w", s, err)
|
||||||
|
}
|
||||||
|
if seen[real] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[real] = true
|
||||||
|
m, err := stat(real)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the socket %s could not be read for its owner and mode: %w", s, err)
|
||||||
|
}
|
||||||
|
a, err := acl(real)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if Writable(m, uid, gids) || grantsByACL(a, uid, gids, true) {
|
||||||
|
ways = append(ways, "it can write the container runtime's socket "+s+", which runs a container as root")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The secrets' ACLs: the bits were judged already.
|
||||||
|
for _, path := range secrets {
|
||||||
|
a, err := acl(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the secret %s's ACL could not be read: %w", path, err)
|
||||||
|
}
|
||||||
|
if grantsByACL(a, uid, gids, false) {
|
||||||
|
ways = append(ways, "an ACL lets it read the secret "+path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setuid programs no package owns.
|
||||||
|
now := time.Now
|
||||||
|
if e.Now != nil {
|
||||||
|
now = e.Now
|
||||||
|
}
|
||||||
|
mountsText, err := read("/proc/mounts")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the mounted filesystems could not be read: %w", err)
|
||||||
|
}
|
||||||
|
mounts := SuidMounts(string(mountsText))
|
||||||
|
if len(mounts) == 0 {
|
||||||
|
return nil, errors.New("no filesystem a setuid program could run from was found in /proc/mounts")
|
||||||
|
}
|
||||||
|
// In the background, the last complete result served while it is fresh (SetuidCache): never a stall. No
|
||||||
|
// fresh complete result is said beside the other ways, which are judged all the same.
|
||||||
|
mountPoints := MountPoints(string(mountsText))
|
||||||
|
unowned, searchedAt, searchErr := e.Cache.Look(now(), mounts,
|
||||||
|
func(ctx context.Context, p *SetuidProgress, checkpoint func()) ([]string, error) {
|
||||||
|
return e.setuidSearch(ctx, p, checkpoint, mountPoints)
|
||||||
|
})
|
||||||
|
var notJudged *SetuidNotJudged
|
||||||
|
if searchErr != nil && !errors.As(searchErr, ¬Judged) {
|
||||||
|
return nil, searchErr
|
||||||
|
}
|
||||||
|
// Counted only where the account can execute it, read now from the paths' modes, never kept with the
|
||||||
|
// search: an account may open its own home to others later without root, and no search would see it.
|
||||||
|
lstat := e.Lstat
|
||||||
|
if lstat == nil {
|
||||||
|
lstat = lstatOf
|
||||||
|
}
|
||||||
|
started := " (search started " + searchedAt.Local().Format("2006-01-02 15:04") + ")"
|
||||||
|
inHomes := 0
|
||||||
|
for _, p := range unowned {
|
||||||
|
if !Executable(p, uid, gids, lstat, acl) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if inAHome(p) {
|
||||||
|
inHomes++ // its path stays in the engine's journal: a home's names are its owner's
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ways = append(ways, "a setuid-root program no package owns, which "+account+" can execute: "+p+started)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case inHomes == 1:
|
||||||
|
ways = append(ways, "a setuid-root program no package owns in an account's home, which "+account+
|
||||||
|
" can execute"+started)
|
||||||
|
case inHomes > 1:
|
||||||
|
ways = append(ways, fmt.Sprintf("%d setuid-root programs no package owns in accounts' homes, which %s can "+
|
||||||
|
"execute%s", inHomes, account, started))
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kernel's protection of links: off, any account may link another's file where root then acts on it.
|
||||||
|
for _, sysctl := range []string{"protected_hardlinks", "protected_symlinks"} {
|
||||||
|
raw, err := read("/proc/sys/fs/" + sysctl)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("fs.%s could not be read: %w", sysctl, err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(raw)) == "0" {
|
||||||
|
what := map[string]string{"protected_hardlinks": "hard links", "protected_symlinks": "symbolic links"}[sysctl]
|
||||||
|
ways = append(ways, fmt.Sprintf("%s to other accounts' files are not protected (fs.%s=0)", what, sysctl))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A container runtime's API on TCP, which any account on the machine reaches.
|
||||||
|
for _, table := range []string{"/proc/net/tcp", "/proc/net/tcp6"} {
|
||||||
|
raw, err := read(table)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) && table == "/proc/net/tcp6" {
|
||||||
|
continue // no IPv6 on this machine
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the listening ports in %s could not be read: %w", table, err)
|
||||||
|
}
|
||||||
|
for _, port := range ListeningPorts(string(raw)) {
|
||||||
|
for _, api := range RuntimeAPIPorts {
|
||||||
|
if port == api {
|
||||||
|
ways = append(ways, fmt.Sprintf("a container runtime's API listens on TCP port %d, which any "+
|
||||||
|
"local account reaches", port))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if notJudged != nil {
|
||||||
|
// The other ways are judged; this one is said as not judged, never as none.
|
||||||
|
return ways, notJudged
|
||||||
|
}
|
||||||
|
return ways, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Executable says whether an account of uid, in the groups gids, can execute path as the kernel decides it:
|
||||||
|
// every directory from the root down searchable for it, and the file executable for it, each by the owner's
|
||||||
|
// bits for its owner, the group's for a member, the others' for anybody else (novox/hq issue 361). It errs
|
||||||
|
// towards a way: a link on the way, an ACL on any of them, or a mode that could not be read is executable.
|
||||||
|
// A path gone is not: there is nothing to execute.
|
||||||
|
func Executable(path string, uid int, gids map[int]bool, lstat func(string) (FileMode, error),
|
||||||
|
acl func(string) ([]ACLEntry, error)) bool {
|
||||||
|
if uid == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
at := "/"
|
||||||
|
steps := []string{at}
|
||||||
|
for _, name := range components(path) {
|
||||||
|
at = filepath.Join(at, name)
|
||||||
|
steps = append(steps, at)
|
||||||
|
}
|
||||||
|
for i, p := range steps {
|
||||||
|
m, err := lstat(p)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if err != nil || m.Perm&fs.ModeSymlink != 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if a, err := acl(p); (err != nil && !errors.Is(err, fs.ErrNotExist)) || len(a) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
bit := fs.FileMode(0o001)
|
||||||
|
switch {
|
||||||
|
case m.UID == uid:
|
||||||
|
bit = 0o100
|
||||||
|
case gids[m.GID]:
|
||||||
|
bit = 0o010
|
||||||
|
}
|
||||||
|
if m.Perm&bit == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if i < len(steps)-1 && !m.Perm.IsDir() {
|
||||||
|
return false // a file on the way: the path names nothing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// homes are where accounts' homes are: a path under one is said by count alone, its name its owner's.
|
||||||
|
var homes = []string{"/home/", "/root/", "/var/home/"}
|
||||||
|
|
||||||
|
func inAHome(path string) bool {
|
||||||
|
for _, h := range homes {
|
||||||
|
if strings.HasPrefix(path, h) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkFiles calls fn for every regular file below dir, through readDir.
|
||||||
|
func walkFiles(readDir func(string) ([]fs.DirEntry, error), dir string, fn func(string) error) error {
|
||||||
|
entries, err := readDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, en := range entries {
|
||||||
|
p := filepath.Join(dir, en.Name())
|
||||||
|
if en.IsDir() {
|
||||||
|
if err := walkFiles(readDir, p, fn); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := fn(p); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CLocale runs a command in the C locale, so what the judge parses — sudo's listing above all — is one
|
||||||
|
// language whatever the machine's is; stdin closed, output captured.
|
||||||
|
func CLocale(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C", "LANGUAGE=C")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if errors.As(err, &exit) {
|
||||||
|
return string(out), fmt.Errorf("%s exited %d: %s", name, exit.ExitCode(), strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
return string(out), fmt.Errorf("%s: %w", name, err)
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user