Compare commits
380
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21eae590e0 | ||
|
|
3b462cac31 | ||
|
|
051a10d14b | ||
|
|
e1aab8a892 | ||
|
|
50a66399f6 | ||
|
|
35f005e53d | ||
|
|
9c9c8a5631 | ||
|
|
bc4d43b508 | ||
|
|
7a5eef32e7 | ||
|
|
5aaafd917f | ||
|
|
aa7c9e8924 | ||
|
|
80f992e367 | ||
|
|
b3fc5b4580 | ||
|
|
6fb27549c9 | ||
|
|
d8227222a7 | ||
|
|
c9d8950fd0 | ||
|
|
df8168f484 | ||
|
|
657cb2b6c8 | ||
|
|
cbea2470d2 | ||
|
|
2dc9e07e88 | ||
|
|
1feda1f533 | ||
|
|
4b9ea0bf11 | ||
|
|
625d627794 | ||
|
|
d630635229 | ||
|
|
29bcaf2be5 | ||
|
|
78a1b5648b | ||
|
|
720efe6339 | ||
|
|
76e25aeee0 | ||
|
|
fb6bf361e4 | ||
|
|
8a8ad7713e | ||
|
|
50acf7c1b4 | ||
|
|
a89aaf5644 | ||
|
|
82d00aeb5e | ||
|
|
2978dd5bed | ||
|
|
1f3a293988 | ||
|
|
bbcc95e66d | ||
|
|
37cdf41088 | ||
|
|
6f2e27f59b | ||
|
|
8d3cffab9a | ||
|
|
73eb3bd11c | ||
|
|
75959dfa25 | ||
|
|
36da33c818 | ||
|
|
ef81dc2712 | ||
|
|
2685b9c749 | ||
|
|
725ea4dba4 | ||
|
|
ffa026f7d8 | ||
|
|
da560f90f8 | ||
|
|
df53a0c59e | ||
|
|
634abead7d | ||
|
|
85999b7efd | ||
|
|
b8642c8b27 | ||
|
|
8f1581f34a | ||
|
|
1e974ce9e8 | ||
|
|
941959198e | ||
|
|
d8b4bb5487 | ||
|
|
078f1e5fa9 | ||
|
|
aa29d8feb8 | ||
|
|
cfde90906f | ||
|
|
8786bb3ba1 | ||
|
|
72f22b11ec | ||
|
|
84a7639042 | ||
|
|
6e4de66df7 | ||
|
|
d4abb82b07 | ||
|
|
42387b660c | ||
|
|
0f1d4c5e0f | ||
|
|
597910c291 | ||
|
|
8fabdc4b91 | ||
|
|
a609ebc8e5 | ||
|
|
a636dc3df3 | ||
|
|
72dd8a719e | ||
|
|
4923e44cf7 | ||
|
|
1d20825d69 | ||
|
|
45f4edbb36 | ||
|
|
c39c3fdddd | ||
|
|
b0f2441dd3 | ||
|
|
62bb25675d | ||
|
|
4104143d16 | ||
|
|
916bdde6a4 | ||
|
|
97cc9b1ccf | ||
|
|
0ae418b43c | ||
|
|
1d3ed6e78e | ||
|
|
f61760f738 | ||
|
|
c46009709d | ||
|
|
b4dbda287b | ||
|
|
e0eecbf3c8 | ||
|
|
c7ae96414f | ||
|
|
9017edaf9e | ||
|
|
75cf83a656 | ||
|
|
059fa5d4c4 | ||
|
|
60c66a3641 | ||
|
|
631f689f40 | ||
|
|
500154b865 | ||
|
|
12b5c62304 | ||
|
|
b2db504ab2 | ||
|
|
e8f93d5db0 | ||
|
|
7c617bb085 | ||
|
|
87388b9046 | ||
|
|
9fafbcfd58 | ||
|
|
2f7ff1c911 | ||
|
|
7ae2ed88c0 | ||
|
|
a86e2ff1fc | ||
|
|
518539f3d1 | ||
|
|
a2272916a0 | ||
|
|
2b2b3e4b8c | ||
|
|
1abae72457 | ||
|
|
0beef7ff9c | ||
|
|
559939aa8c | ||
|
|
87fd0cd8b9 | ||
|
|
279ac41087 | ||
|
|
b0bdcd02c9 | ||
|
|
211186edd7 | ||
|
|
5ceb6e9e38 | ||
|
|
d023a25849 | ||
|
|
ebe1248f0b | ||
|
|
8377f8a878 | ||
|
|
796fb69d17 | ||
|
|
236ce2fe12 | ||
|
|
bbb9e37df4 | ||
|
|
1bf98663c6 | ||
|
|
541c2e02d9 | ||
|
|
fd4931a752 | ||
|
|
64cb7ecc53 | ||
|
|
10bbbadcf2 | ||
|
|
3dfc3533be | ||
|
|
da2aa91832 | ||
|
|
1332f8b523 | ||
|
|
af487d38ed | ||
|
|
cea09d8a36 | ||
|
|
b4e0518282 | ||
|
|
d0a69afbb1 | ||
|
|
4d3c901b99 | ||
|
|
680da05c4c | ||
|
|
d8af436536 | ||
|
|
1f9b130225 | ||
|
|
5ab349df6c | ||
|
|
c50a7d059a | ||
|
|
c275919f11 | ||
|
|
4a3d7b60a1 | ||
|
|
f3983ed473 | ||
|
|
480556a329 | ||
|
|
ac29f20517 | ||
|
|
f86a1f2a36 | ||
|
|
dad2708805 | ||
|
|
80cdcbfa2f | ||
|
|
4b93dbb04e | ||
|
|
ddcd790304 | ||
|
|
34269df920 | ||
|
|
1af1a7b002 | ||
|
|
b5644829d3 | ||
|
|
cd3e444a76 | ||
|
|
ce989b001a | ||
|
|
7a60771165 | ||
|
|
6cd4aedb7c | ||
|
|
4c4abb1b66 | ||
|
|
b4ab61c0aa | ||
|
|
39cd545814 | ||
|
|
5ef6287ce4 | ||
|
|
7cd9b611d2 | ||
|
|
1f79a4d52a | ||
|
|
65c2d76506 | ||
|
|
d3fa102e4d | ||
|
|
fd908a444d | ||
|
|
b4e7ffd89c | ||
|
|
ad671183bf | ||
|
|
d3d4dbc8ed | ||
|
|
9d09aeddb1 | ||
|
|
4438045b8c | ||
|
|
9c8d64e28c | ||
|
|
31f6c635f5 | ||
|
|
348f9631fe | ||
|
|
eb50bf1f48 | ||
|
|
b88c6dac5b | ||
|
|
37a16a4e98 | ||
|
|
4915e67520 | ||
|
|
8acad77d33 | ||
|
|
37a1f2be53 | ||
|
|
46505d42d8 | ||
|
|
1c4739698a | ||
|
|
a198a959fe | ||
|
|
6da45bc090 | ||
|
|
05dd17ebaa | ||
|
|
d47d231ec3 | ||
|
|
e71b006d85 | ||
|
|
d2f326dff4 | ||
|
|
725eea8a68 | ||
|
|
3f39b60d9a | ||
|
|
e40ace34a6 | ||
|
|
880de92bc8 | ||
|
|
6ec562e964 | ||
|
|
2c807a3800 | ||
|
|
2770a55b61 | ||
|
|
9d83987815 | ||
|
|
af280050d6 | ||
|
|
87a30055a6 | ||
|
|
39a8d8d7ee | ||
|
|
d9d1119a72 | ||
|
|
e9ba05bd9a | ||
|
|
7b08f25d4a | ||
|
|
a46e52000b | ||
|
|
aad5c6ceea | ||
|
|
c295176048 | ||
|
|
ea4e22577c | ||
|
|
6061c775d5 | ||
|
|
8f5667cca5 | ||
|
|
b5495d0cc8 | ||
|
|
c5bec10bb5 | ||
|
|
aaca1c6ca9 | ||
|
|
5450707aaf | ||
|
|
79c2fc7fe1 | ||
|
|
cb3db6c902 | ||
|
|
6146c8a76d | ||
|
|
4492bc3013 | ||
|
|
929b1ce13e | ||
|
|
55a88cdba4 | ||
|
|
29f6039fb1 | ||
|
|
57a18a4783 | ||
|
|
b0a275c126 | ||
|
|
d7ffc9cca0 | ||
|
|
38ae52e999 | ||
|
|
11acf04976 | ||
|
|
63322ef06f | ||
|
|
4fa457b9bb | ||
|
|
b665156b0f | ||
|
|
0b2260d9fb | ||
|
|
f16efbef4f | ||
|
|
ede8994799 | ||
|
|
d31b41bb7e | ||
|
|
b4584b0313 | ||
|
|
c6c046283d | ||
|
|
d683a78d9c | ||
|
|
ed7c6c4fb6 | ||
|
|
cd66db54bf | ||
|
|
9f01b9e79a | ||
|
|
e34fcbeda3 | ||
|
|
32699e35b8 | ||
|
|
45688f196e | ||
|
|
85fd46b4cc | ||
|
|
f8212dc203 | ||
|
|
d7214bb576 | ||
|
|
11048b4a2d | ||
|
|
99512eade4 | ||
|
|
ee54bae09f | ||
|
|
75986b2960 | ||
|
|
3daedac1bb | ||
|
|
313c1997a4 | ||
|
|
e6806d5698 | ||
|
|
d0f384610c | ||
|
|
8d4b2cb1c6 | ||
|
|
4f433bdaf8 | ||
|
|
91cd016f65 | ||
|
|
cdddeee4ea | ||
|
|
3e019c5d7e | ||
|
|
fa4ccbc0a9 | ||
|
|
76aab05289 | ||
|
|
4b2d5aeaab | ||
|
|
68dd69f4ef | ||
|
|
3b61a05532 | ||
|
|
6be2e91da8 | ||
|
|
905c34a537 | ||
|
|
cc1a942235 | ||
|
|
93bfdd9e2f | ||
|
|
1f68f80760 | ||
|
|
ff4d042be5 | ||
|
|
4f24cb0751 | ||
|
|
0d86833123 | ||
|
|
715bc088c4 | ||
|
|
ab5f767372 | ||
|
|
40b1307839 | ||
|
|
a3a11842bf | ||
|
|
bcac2d6d1c | ||
|
|
4d597eb9dc | ||
|
|
c73641c117 | ||
|
|
47b3ae7528 | ||
|
|
e5e3d7dce6 | ||
|
|
7843edb1eb | ||
|
|
2bf608fc5f | ||
|
|
b0bfdeb0d1 | ||
|
|
b00a826681 | ||
|
|
25e5d00aba | ||
|
|
a4e3a9cbe0 | ||
|
|
d618cbb9ca | ||
|
|
6b929c9057 | ||
|
|
550bb4fdd1 | ||
|
|
f62c2ff380 | ||
|
|
972ecd105d | ||
|
|
7bf0e991d8 | ||
|
|
2339ac5af7 | ||
|
|
692f18ef3b | ||
|
|
5bc05201ef | ||
|
|
c5dc10fa8c | ||
|
|
9e726d4c41 | ||
|
|
7fe569ef40 | ||
|
|
aff9d6ae6a | ||
|
|
40e1e087e5 | ||
|
|
eeeb2056ae | ||
|
|
e1a182bf09 | ||
|
|
553cf50da7 | ||
|
|
4e318c8591 | ||
|
|
c9b98477b7 | ||
|
|
bd9764ef9a | ||
|
|
bb85684b2a | ||
|
|
17ba95bca6 | ||
|
|
9ad22ad1e1 | ||
|
|
4aa77a4237 | ||
|
|
4d95d8fc1e | ||
|
|
d6c3fc1e40 | ||
|
|
285099a940 | ||
|
|
80a7cee6f3 | ||
|
|
e0d7644bb6 | ||
|
|
376a8b0fb5 | ||
|
|
6a39f43404 | ||
|
|
c973b31820 | ||
|
|
7f626147be | ||
|
|
080aed6855 | ||
|
|
4f67143a78 | ||
|
|
1572ab9b25 | ||
|
|
b0b7dfc5eb | ||
|
|
bf4ce7ef7b | ||
|
|
052fb473d0 | ||
|
|
bff850f798 | ||
|
|
7a7db0166a | ||
|
|
237a534ca4 | ||
|
|
dd458cb155 | ||
|
|
6c09be4a70 | ||
|
|
448f1ef481 | ||
|
|
093967af9b | ||
|
|
be749c9513 | ||
|
|
4026392a6e | ||
|
|
d632071317 | ||
|
|
5fe7e3e6ba | ||
|
|
be24f5ac38 | ||
|
|
f2eb548204 | ||
|
|
e6de426303 | ||
|
|
5fab8f118e | ||
|
|
d5237091a6 | ||
|
|
1a35e5cf4d | ||
|
|
e2fad6a50a | ||
|
|
04ab9bc603 | ||
|
|
333ab9de22 | ||
|
|
e84b9d13c4 | ||
|
|
9c36d4edd4 | ||
|
|
4c137795a8 | ||
|
|
281f586621 | ||
|
|
843da56d09 | ||
|
|
2d34ff468b | ||
|
|
230230ce0e | ||
|
|
34ce2895dd | ||
|
|
0aa319154e | ||
|
|
90ff598207 | ||
|
|
1d1d5ec443 | ||
|
|
3112692def | ||
|
|
4915ff5d29 | ||
|
|
dd4c53eefb | ||
|
|
32dc74788e | ||
|
|
1fb32928ae | ||
|
|
0b2d393b11 | ||
|
|
134baf0e3b | ||
|
|
7c78fd2e0c | ||
|
|
9239177efd | ||
|
|
a1d98c2438 | ||
|
|
f469eb387a | ||
|
|
d221c63379 | ||
|
|
68e7a2ddbf | ||
|
|
ee75d4327c | ||
|
|
640455999e | ||
|
|
45084af849 | ||
|
|
7f4cc9e2ec | ||
|
|
06963203ff | ||
|
|
0697b2994e | ||
|
|
308d052ff6 | ||
|
|
1885f176f1 | ||
|
|
6bdf958cef | ||
|
|
4797790959 | ||
|
|
7aad491321 | ||
|
|
6118c790e6 | ||
|
|
6f781abb57 | ||
|
|
350aca05fb | ||
|
|
b83a1abe5f | ||
|
|
9c57f7afb3 | ||
|
|
2d2eb5dffb |
+229
-39
@@ -1,46 +1,236 @@
|
||||
node_modules/
|
||||
dist/
|
||||
|
||||
# Go tool bundles built in place (go build in a module's cmd/<name>-tools) are build output.
|
||||
modules/pacman/pacman-tools
|
||||
modules/slack/cmd/slack-tools/slack-tools
|
||||
modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
|
||||
modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
modules/claude-code/cmd/claude-code/claude-code
|
||||
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
|
||||
modules/messenger/messenger
|
||||
modules/claude-code/claude-code
|
||||
modules/mesh-watcher/mesh-watcher
|
||||
modules/dnsmasq/dnsmasq
|
||||
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
modules/mesh-issues/mesh-issues
|
||||
modules/mounts/cmd/mounts/mounts
|
||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-reads/mesh-reads
|
||||
modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
modules/mesh-files/mesh-files
|
||||
modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
|
||||
# The provisioner client built in place (issue 671).
|
||||
modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
*.test
|
||||
|
||||
# binaries a local go build leaves in a module root
|
||||
# Every program a local `go build` leaves, in its main package's directory and at its Go module's root, each anchored to
|
||||
# its one path so it never hides source (issue 1577: twice in one milestone a module's program was committed). The
|
||||
# checks/gitignored test refuses a main package whose program no ignore file names: add its line here.
|
||||
/checks/agenthome/agenthome
|
||||
/checks/config/config
|
||||
/checks/deps/deps
|
||||
/checks/environment/environment
|
||||
/checks/imagebuild/imagebuild
|
||||
/checks/imageplace/imageplace
|
||||
/checks/locks/locks
|
||||
/checks/mailstore/mailstore
|
||||
/checks/noplacement/noplacement
|
||||
/checks/owned/owned
|
||||
/checks/particulars/particulars
|
||||
/checks/paths/paths
|
||||
/checks/predefined/predefined
|
||||
/checks/readable/readable
|
||||
/checks/removedpackages/removedpackages
|
||||
/checks/secrets/secrets
|
||||
/checks/sshdir/sshdir
|
||||
/checks/templates/templates
|
||||
/checks/toolchains/toolchains
|
||||
/checks/words/words
|
||||
/modules/adwaita/adwaita-tools
|
||||
/modules/adwaita/cmd/adwaita-tools/adwaita-tools
|
||||
/modules/artifact-store-tools/artifact-store-tools
|
||||
/modules/artifact-store-tools/cmd/artifact-store-tools/artifact-store-tools
|
||||
/modules/asus-zephyrus-g14/cmd/zephyrus/zephyrus
|
||||
/modules/asus-zephyrus-g14/zephyrus
|
||||
/modules/avahi/avahi-tools
|
||||
/modules/avahi/cmd/avahi-tools/avahi-tools
|
||||
/modules/blueman/blueman-tools
|
||||
/modules/blueman/cmd/blueman-tools/blueman-tools
|
||||
/modules/bluetooth/bluetooth-tools
|
||||
/modules/bluetooth/cmd/bluetooth-tools/bluetooth-tools
|
||||
/modules/brother/brother
|
||||
/modules/brother/cmd/brother/brother
|
||||
/modules/claude-code/claude-code
|
||||
/modules/claude-code/cmd/claude-code/claude-code
|
||||
/modules/claude-licence-manager/claude-licence-manager
|
||||
/modules/claude-licence-manager/cmd/claude-licence-manager/claude-licence-manager
|
||||
/modules/clipmenu/clipmenu-tools
|
||||
/modules/clipmenu/cmd/clipmenu-tools/clipmenu-tools
|
||||
/modules/cups/cmd/cups-tools/cups-tools
|
||||
/modules/cups/cups-tools
|
||||
/modules/dbus/cmd/dbus-tools/dbus-tools
|
||||
/modules/dbus/dbus-tools
|
||||
/modules/desk-channel/cmd/desk-channel/desk-channel
|
||||
/modules/desk-channel/desk-channel
|
||||
/modules/disk-load/cmd/disk-load/disk-load
|
||||
/modules/disk-load/disk-load
|
||||
/modules/distribution/cmd/distribution-tools/distribution-tools
|
||||
/modules/distribution/cmd/distribution/distribution
|
||||
/modules/distribution/distribution
|
||||
/modules/distribution/distribution-tools
|
||||
/modules/dmenu/cmd/dmenu-tools/dmenu-tools
|
||||
/modules/dmenu/dmenu-tools
|
||||
/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
/modules/dnsmasq/dnsmasq-tools
|
||||
/modules/docker-compose/cmd/docker-compose-tools/docker-compose-tools
|
||||
/modules/docker-compose/docker-compose-tools
|
||||
/modules/docker/cmd/docker-tools/docker-tools
|
||||
/modules/docker/docker-tools
|
||||
/modules/dunst/cmd/dunst-tools/dunst-tools
|
||||
/modules/dunst/dunst-tools
|
||||
/modules/fail2ban/cmd/fail2ban-tools/fail2ban-tools
|
||||
/modules/fail2ban/fail2ban-tools
|
||||
/modules/feh/cmd/feh-tools/feh-tools
|
||||
/modules/feh/feh-tools
|
||||
/modules/flatpak/cmd/flatpak-tools/flatpak-tools
|
||||
/modules/flatpak/flatpak-tools
|
||||
/modules/fonts/cmd/fonts-tools/fonts-tools
|
||||
/modules/fonts/fonts-tools
|
||||
/modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
/modules/forticlient/forticlient-tools
|
||||
/modules/gitea/agent-teams
|
||||
/modules/gitea/agent-token
|
||||
/modules/gitea/cmd/agent-teams/agent-teams
|
||||
/modules/gitea/cmd/agent-token/agent-token
|
||||
/modules/gitea/cmd/forge-account-provider/forge-account-provider
|
||||
/modules/gitea/cmd/forge/forge
|
||||
/modules/gitea/cmd/package-registry/package-registry
|
||||
/modules/gitea/cmd/repositories/repositories
|
||||
/modules/gitea/forge
|
||||
/modules/gitea/forge-account-provider
|
||||
/modules/gitea/package-registry
|
||||
/modules/gitea/repositories
|
||||
/modules/gnome-keyring/cmd/gnome-keyring-tools/gnome-keyring-tools
|
||||
/modules/gnome-keyring/gnome-keyring-tools
|
||||
/modules/hostname/cmd/hostname-tools/hostname-tools
|
||||
/modules/hostname/hostname-tools
|
||||
/modules/i3/cmd/i3-tools/i3-tools
|
||||
/modules/i3/i3-tools
|
||||
/modules/i3status-rust/cmd/i3status-rust-tools/i3status-rust-tools
|
||||
/modules/i3status-rust/i3status-rust-tools
|
||||
/modules/kernel/cmd/kernel-tools/kernel-tools
|
||||
/modules/kernel/kernel-tools
|
||||
/modules/keycloak/cmd/keycloak-provider/keycloak-provider
|
||||
/modules/keycloak/keycloak-provider
|
||||
/modules/lemurs/cmd/lemurs-tools/lemurs-tools
|
||||
/modules/lemurs/lemurs-tools
|
||||
/modules/localization/cmd/localization-tools/localization-tools
|
||||
/modules/localization/localization-tools
|
||||
/modules/logrotate/cmd/logrotate-tools/logrotate-tools
|
||||
/modules/logrotate/logrotate-tools
|
||||
/modules/mailu/automx
|
||||
/modules/mailu/cmd/automx/automx
|
||||
/modules/memory-pressure/cmd/memory-pressure/memory-pressure
|
||||
/modules/memory-pressure/memory-pressure
|
||||
/modules/mesh-build-go/buildstep/cmd/mesh-pack/mesh-pack
|
||||
/modules/mesh-build-go/buildstep/cmd/seatgen/seatgen
|
||||
/modules/mesh-build-go/buildstep/mesh-pack
|
||||
/modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
/modules/mesh-build-image/mesh-build-image
|
||||
/modules/mesh-build-node/cmd/mesh-build-node/mesh-build-node
|
||||
/modules/mesh-build-node/mesh-build-node
|
||||
/modules/mesh-cli/cmd/mesh-cli/mesh-cli
|
||||
/modules/mesh-cli/mesh-cli
|
||||
/modules/mesh-conditions/cmd/mesh-conditions/mesh-conditions
|
||||
/modules/mesh-conditions/mesh-conditions
|
||||
/modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
/modules/mesh-config/mesh-config
|
||||
/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mesh-delivery-gate
|
||||
/modules/mesh-delivery-gate/mesh-delivery-gate
|
||||
modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
|
||||
# Built by their modules' build step (issue 1577), never committed.
|
||||
modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
modules/networkmanager/uplink-tools
|
||||
checks/words/words
|
||||
modules/mesh-cli/mesh-cli
|
||||
modules/mesh-cli/cmd/mesh-cli/mesh-cli
|
||||
/modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
/modules/mesh-files/mesh-files
|
||||
/modules/mesh-hand-acts/cmd/mesh-hand-acts/mesh-hand-acts
|
||||
/modules/mesh-hand-acts/mesh-hand-acts
|
||||
/modules/mesh-healers/cmd/mesh-healers/mesh-healers
|
||||
/modules/mesh-healers/mesh-healers
|
||||
/modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
/modules/mesh-issues/mesh-issues
|
||||
/modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
/modules/mesh-knowledge/mesh-knowledge
|
||||
/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/mesh-mail-notifier
|
||||
/modules/mesh-mail-notifier/mesh-mail-notifier
|
||||
/modules/mesh-mail-reader/cmd/mesh-mail-reader/mesh-mail-reader
|
||||
/modules/mesh-mail-reader/mesh-mail-reader
|
||||
/modules/mesh-mail-web/cmd/mesh-mail-web/mesh-mail-web
|
||||
/modules/mesh-mail-web/mesh-mail-web
|
||||
/modules/mesh-mail/cmd/mesh-mail/mesh-mail
|
||||
/modules/mesh-mail/mesh-mail
|
||||
/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
/modules/mesh-provisioner-client/mesh-provisioner-client
|
||||
/modules/mesh-provisioner/cmd/mesh-provisioner/mesh-provisioner
|
||||
/modules/mesh-provisioner/mesh-provisioner
|
||||
/modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
/modules/mesh-reads/mesh-reads
|
||||
/modules/mesh-self-check/cmd/mesh-self-check/mesh-self-check
|
||||
/modules/mesh-self-check/mesh-self-check
|
||||
/modules/mesh-vault/cmd/mesh-vault/mesh-vault
|
||||
/modules/mesh-vault/mesh-vault
|
||||
/modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
/modules/mesh-watcher/mesh-watcher
|
||||
/modules/messenger/cmd/messenger/messenger
|
||||
/modules/messenger/messenger
|
||||
/modules/minio/cmd/object-store/object-store
|
||||
/modules/minio/object-store
|
||||
/modules/mounts/cmd/mounts/mounts
|
||||
/modules/mounts/mounts
|
||||
/modules/nats/cmd/mesh-nats-snapshot/mesh-nats-snapshot
|
||||
/modules/nats/cmd/nats-tools/nats-tools
|
||||
/modules/nats/mesh-nats-snapshot
|
||||
/modules/nats/nats-tools
|
||||
/modules/netcheck/cmd/netcheck/netcheck
|
||||
/modules/netcheck/netcheck
|
||||
/modules/networkmanager/cmd/uplink-tools/uplink-tools
|
||||
/modules/networkmanager/uplink-tools
|
||||
/modules/nextcloud-client/cmd/nextcloud-client-tools/nextcloud-client-tools
|
||||
/modules/nextcloud-client/nextcloud-client-tools
|
||||
/modules/nfs-server/cmd/nfs-server/nfs-server
|
||||
/modules/nfs-server/nfs-server
|
||||
/modules/nm-applet/cmd/nm-applet-tools/nm-applet-tools
|
||||
/modules/nm-applet/nm-applet-tools
|
||||
/modules/only-office/cmd/only-office/only-office
|
||||
/modules/only-office/only-office
|
||||
/modules/openrazer/cmd/openrazer-tools/openrazer-tools
|
||||
/modules/openrazer/openrazer-tools
|
||||
/modules/pacman/cmd/pacman-tools/pacman-tools
|
||||
/modules/pacman/pacman-tools
|
||||
/modules/picom/cmd/picom-tools/picom-tools
|
||||
/modules/picom/picom-tools
|
||||
/modules/polychromatic/cmd/polychromatic-tools/polychromatic-tools
|
||||
/modules/polychromatic/polychromatic-tools
|
||||
/modules/postgres/cmd/postgres-provider/postgres-provider
|
||||
/modules/postgres/postgres-provider
|
||||
/modules/power/cmd/power/power
|
||||
/modules/power/power
|
||||
/modules/records/cmd/records/records
|
||||
/modules/records/records
|
||||
/modules/restic/cmd/restic-backups/restic-backups
|
||||
/modules/restic/restic-backups
|
||||
/modules/rofi/cmd/rofi-tools/rofi-tools
|
||||
/modules/rofi/rofi-tools
|
||||
/modules/route-adapter/cmd/route-provider/route-provider
|
||||
/modules/route-adapter/route-provider
|
||||
/modules/screen-lock/cmd/screen-lock-tools/screen-lock-tools
|
||||
/modules/screen-lock/screen-lock-tools
|
||||
/modules/sensors/cmd/sensors/sensors
|
||||
/modules/sensors/sensors
|
||||
/modules/slack/cmd/slack-tools/slack-tools
|
||||
/modules/slack/slack-tools
|
||||
/modules/snapd/cmd/snapd-tools/snapd-tools
|
||||
/modules/snapd/snapd-tools
|
||||
/modules/ssh-client/cmd/ssh-client-tools/ssh-client-tools
|
||||
/modules/ssh-client/ssh-client-tools
|
||||
/modules/sudo/cmd/sudo-tools/sudo-tools
|
||||
/modules/sudo/sudo-tools
|
||||
/modules/systemd-networkd/cmd/uplink-tools/uplink-tools
|
||||
/modules/systemd-networkd/uplink-tools
|
||||
/modules/systemd-resolved/cmd/resolver-tools/resolver-tools
|
||||
/modules/systemd-resolved/resolver-tools
|
||||
/modules/systemd/cmd/systemd-tools/systemd-tools
|
||||
/modules/systemd/systemd-tools
|
||||
/modules/telegram/cmd/telegram/telegram
|
||||
/modules/telegram/telegram
|
||||
/modules/time-sync/cmd/time-sync-tools/time-sync-tools
|
||||
/modules/time-sync/time-sync-tools
|
||||
/modules/triggerhappy/cmd/triggerhappy-tools/triggerhappy-tools
|
||||
/modules/triggerhappy/triggerhappy-tools
|
||||
/modules/xclip/cmd/xclip-tools/xclip-tools
|
||||
/modules/xclip/xclip-tools
|
||||
/modules/xdg/cmd/xdg-tools/xdg-tools
|
||||
/modules/xdg/xdg-tools
|
||||
/modules/xorg/cmd/xorg/xorg
|
||||
/modules/xorg/xorg
|
||||
/modules/xterm/cmd/xterm-tools/xterm-tools
|
||||
/modules/xterm/xterm-tools
|
||||
/modules/zfs/cmd/zfs-tools/zfs-tools
|
||||
/modules/zfs/zfs-tools
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/config
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -10,6 +10,9 @@
|
||||
// (`mesh-controller.<name>`, `seat:mesh-controller.<verb>`, `mesh-controller_<bucket>`), anywhere in the manifest;
|
||||
// - a manifest string that is the bare module name `mesh-controller` outside the module's own `config`, where a value
|
||||
// is the module's data (an issue owner naming the controller's repository is no dependency), not an address;
|
||||
// - a go.mod of a module or of a check requiring the controller's repository (its module path, github.com/novox/mesh-controller,
|
||||
// or the forge's git.novox.be/novox/mesh-controller): the repository is archived once its last module is delivered
|
||||
// elsewhere (novox/hq ADR 0593 §11), so nothing here may be built from it;
|
||||
// - a line of any module's file naming one of the controller's buckets (`mesh-controller_`): its buckets are removed
|
||||
// after the controller's end (ADR 0590 §6), so a reader of one reads nothing. A test (`_test.go`, `.test.ts`) reads
|
||||
// no live bucket, its names are its own data, and a `testdata` directory is a copy of text from elsewhere (a design,
|
||||
@@ -40,6 +43,9 @@ const (
|
||||
// address is a string naming the controller's seat, one of its states, verbs or events, or one of its buckets.
|
||||
var address = regexp.MustCompile(`(^|[^A-Za-z0-9_-])(seat:)?mesh-controller([._][A-Za-z0-9])`)
|
||||
|
||||
// repository is a go.mod line naming the controller's repository by either of its module paths.
|
||||
var repository = regexp.MustCompile(`(github\.com|git\.novox\.be)/novox/mesh-controller(/|\s|$)`)
|
||||
|
||||
// Refusals is each reason the check gives over the tree at root, sorted; none when it passes.
|
||||
func Refusals(root string) ([]string, error) {
|
||||
var out []string
|
||||
@@ -65,6 +71,26 @@ func Refusals(root string) ([]string, error) {
|
||||
"consumes, waits on or uses it; name the seat that holds the job now (novox/hq ADR 0590 §5)", module, s))
|
||||
}
|
||||
}
|
||||
mods, err := filepath.Glob(filepath.Join(root, "*", "*", "go.mod"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(mods)
|
||||
for _, f := range mods {
|
||||
if rel, _ := filepath.Rel(root, f); !strings.HasPrefix(rel, "modules"+string(filepath.Separator)) && !strings.HasPrefix(rel, "checks"+string(filepath.Separator)) {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for n, l := range strings.Split(string(raw), "\n") {
|
||||
if repository.MatchString(l) {
|
||||
rel, _ := filepath.Rel(root, f)
|
||||
out = append(out, fmt.Sprintf("%s:%d: requires the controller's repository: it is archived once its last module is built elsewhere, so nothing here is built from it (novox/hq ADR 0593 §11)", filepath.ToSlash(rel), n+1))
|
||||
}
|
||||
}
|
||||
}
|
||||
buckets, err := bucketLines(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -33,10 +33,13 @@ func TestRefusalsJudgeATree(t *testing.T) {
|
||||
{"a requirement", map[string]string{"modules/d/module.json": `{"module":"d","requires":[{"name":"x","from":"mesh-controller"}]}`}, `d: names "mesh-controller"`},
|
||||
{"the controller's own module", map[string]string{"modules/mesh-controller/module.json": `{"module":"mesh-controller"}`}, "mesh-controller: the controller is no module of the catalogue"},
|
||||
{"a bucket in code", map[string]string{"modules/e/cmd/e/main.go": "package main\nconst b = \"mesh-controller_assignments\"\n"}, "modules/e/cmd/e/main.go:2: names the controller's bucket"},
|
||||
{"a requirement of the controller's repository", map[string]string{"modules/h/go.mod": "module h\n\nrequire github.com/novox/mesh-controller v0.0.0\n"}, "modules/h/go.mod:3: requires the controller's repository"},
|
||||
{"its forge path in a check", map[string]string{"checks/i/go.mod": "module i\n\nrequire (\n\tgit.novox.be/novox/mesh-controller/modules/route-proxy v0.1.0\n)\n"}, "checks/i/go.mod:4"},
|
||||
{"another repository of the forge passes", map[string]string{"modules/j/go.mod": "module j\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.0\n"}, ""},
|
||||
{"a bucket in a test is its data", map[string]string{"modules/e/cmd/e/main_test.go": "package main\n// mesh-controller_lease\n"}, ""},
|
||||
{"a fixture copied from elsewhere", map[string]string{"modules/e/cmd/e/testdata/x.md": "mesh-controller_rollouts\n"}, ""},
|
||||
{"an owner and prose pass", map[string]string{"modules/f/module.json": `{"module":"f","config":{"owners":{"default":["mesh-controller","mesh-controller modules/route-proxy"]}},"why":"the mesh-controller module is gone"}`}, ""},
|
||||
{"the seat's successors pass", map[string]string{"modules/g/module.json": `{"module":"g","reads":["mesh-placement.placement"],"invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
{"the seat's successors pass", map[string]string{"modules/g/module.json": `{"module":"g","reads":["mesh-module-assigner.assignments"],"invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
@@ -2,4 +2,4 @@ module checks/credentialform
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
+13
-2
@@ -111,13 +111,24 @@ func forgeRequirements(dir string) ([]requirement, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %s", lastLine(out))
|
||||
}
|
||||
var mod struct{ Require []requirement }
|
||||
var mod struct {
|
||||
Require []requirement
|
||||
Replace []struct{ Old, New requirement }
|
||||
}
|
||||
if err := json.Unmarshal(out, &mod); err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %w", err)
|
||||
}
|
||||
// A requirement the go.mod replaces to a directory is a package of the module's own repository (novox/hq ADR 0593 §1): it
|
||||
// is in the tree under check, so it is fetched from nowhere.
|
||||
local := map[string]bool{}
|
||||
for _, r := range mod.Replace {
|
||||
if r.New.Version == "" {
|
||||
local[r.Old.Path] = true
|
||||
}
|
||||
}
|
||||
var on []requirement
|
||||
for _, r := range mod.Require {
|
||||
if strings.HasPrefix(r.Path, forge) {
|
||||
if strings.HasPrefix(r.Path, forge) && !local[r.Path] {
|
||||
on = append(on, r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,3 +175,28 @@ func TestTheRefOfAVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0593 §1: a module requires a sibling package of its own repository and replaces it to the directory beside it; that
|
||||
// package is no fetch from the forge (it is in the tree under check), so it is never looked for in a clone beside the check,
|
||||
// and the module builds with the SDK taken from there.
|
||||
func TestASiblingPackageReplacedToItsDirectoryIsNotFetched(t *testing.T) {
|
||||
offline(t)
|
||||
beside, commit := sdkBeside(t)
|
||||
version := "v0.1.14-0.20261012082113-" + commit[:12]
|
||||
root := t.TempDir()
|
||||
sibling := "git.novox.be/novox/mesh-catalog/modules/builder/runner"
|
||||
write(t, filepath.Join(root, "builder", "runner", "go.mod"), "module "+sibling+"\n\ngo 1.22\n\nrequire git.novox.be/novox/mesh-sdk/go "+version+"\n")
|
||||
write(t, filepath.Join(root, "builder", "runner", "run.go"), "package runner\n\nimport \"git.novox.be/novox/mesh-sdk/go/sdk\"\n\nfunc Run() string { return sdk.Hello() }\n")
|
||||
dir := filepath.Join(root, "user")
|
||||
write(t, filepath.Join(dir, "go.mod"), "module git.novox.be/novox/mesh-catalog/modules/user\n\ngo 1.22\n\nrequire (\n\t"+sibling+" v0.0.0\n"+
|
||||
"\tgit.novox.be/novox/mesh-sdk/go "+version+"\n)\n\nreplace "+sibling+" => ../builder/runner\n")
|
||||
write(t, filepath.Join(dir, "main.go"), "package main\n\nimport \""+sibling+"\"\n\nfunc main() { println(runner.Run()) }\n")
|
||||
modfile, err := resolve(dir, beside, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if modfile == "" {
|
||||
t.Fatal("resolve said the module fetches its dependencies, with nothing to fetch the SDK from")
|
||||
}
|
||||
builds(t, dir, modfile)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/environment
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,252 @@
|
||||
// Command environment refuses a module's code reading a MESH_* word of its environment that nothing gives it: not its
|
||||
// manifest, and not the node-engine or the tool runner for every process (the lead's ruling on A3n, issue 1966).
|
||||
//
|
||||
// A MESH_* word is the mesh's contract with a module's process: the manifest sets it (an `env` of a build artifact or a
|
||||
// resource, an `environment`'s `variables`, a `NAME=` line of a file the module ships for an env-file), or the mesh gives
|
||||
// it to every process (Given). A word read and set by nothing is a contract nobody keeps: the program runs on its
|
||||
// default or on nothing, and says so to nobody (MESH_REGISTRY_REACH answered every published reference without its
|
||||
// registry; MESH_CONFIG_CALLERS held every window shut). Found here, it is refused: the code reads what the mesh gives
|
||||
// (a manifest word, a config value through its tools, a seat's name), or the manifest sets it.
|
||||
//
|
||||
// A read is `os.Getenv` or `os.LookupEnv` of a literal or of a constant of the same directory (Go), and `env.MESH_*` or
|
||||
// `env["MESH_*"]` (TypeScript, `process.env` included); tests, test directories, node_modules and dist are not read. The
|
||||
// modules still reading such a word are listed in the file `unset` beside this check, `<module> <WORD>` a line, until
|
||||
// each is fixed: a line no longer found is refused, so the list only shrinks.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Given is every MESH_* word the mesh gives a module's processes whatever their manifest says, and who gives it.
|
||||
var Given = map[string]string{
|
||||
// The tool runner, to every bundle and to a module's own runtime (mesh-tools node-tools runtime.BundleEnv).
|
||||
"MESH_MODULE": "the tool runner",
|
||||
"MESH_SERVED_MODULE": "the tool runner",
|
||||
"MESH_NODE": "the tool runner",
|
||||
// The node-engine, to the tool runner it runs as the operator's account; its bundles inherit them (mesh-host
|
||||
// compose/resolution runtime.go).
|
||||
"MESH_OPERATOR_ACCOUNT": "the node-engine",
|
||||
"MESH_OPERATOR_HOME": "the node-engine",
|
||||
// The node-engine, to every module that yields to the node's builds (ADR 0553 §3; catalogue.BuildCPUShareWord).
|
||||
"MESH_BUILD_CPU_SHARE": "the node-engine",
|
||||
}
|
||||
|
||||
const listFile = "unset"
|
||||
|
||||
var (
|
||||
goRead = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*(?:"(MESH_[A-Z0-9_]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*\)`)
|
||||
goConst = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:string\s*)?=\s*"(MESH_[A-Z0-9_]+)"`)
|
||||
tsRead = regexp.MustCompile(`\benv(?:\.|\[\s*["'])(MESH_[A-Z0-9_]+)`)
|
||||
envLine = regexp.MustCompile(`(?m)^\s*(?:export\s+)?(MESH_[A-Z0-9_]+)=`)
|
||||
)
|
||||
|
||||
// skipped are the directories whose code no process of the module runs.
|
||||
var skipped = map[string]bool{"node_modules": true, "dist": true, "test": true, "testdata": true, ".git": true}
|
||||
|
||||
// Check answers, per module, each MESH_* word its code reads that its manifest does not set and the mesh does not
|
||||
// give, with the files reading it.
|
||||
func Check(root string) (map[string]map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
out := map[string]map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
dir := filepath.Dir(path)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var manifest any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
set := map[string]bool{}
|
||||
setBy(manifest, set)
|
||||
reads := map[string]map[string]bool{}
|
||||
consts := map[string]map[string]string{} // directory -> constant -> word
|
||||
var goFiles []string
|
||||
err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if p != dir && skipped[d.Name()] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
text, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
rel = filepath.ToSlash(rel)
|
||||
switch {
|
||||
case strings.HasSuffix(name, "_test.go"), strings.HasSuffix(name, ".test.ts"), strings.HasSuffix(name, ".d.ts"):
|
||||
case strings.HasSuffix(name, ".go"):
|
||||
goFiles = append(goFiles, p)
|
||||
c := consts[filepath.Dir(p)]
|
||||
if c == nil {
|
||||
c = map[string]string{}
|
||||
consts[filepath.Dir(p)] = c
|
||||
}
|
||||
for _, m := range goConst.FindAllStringSubmatch(string(text), -1) {
|
||||
c[m[1]] = m[2]
|
||||
}
|
||||
case strings.HasSuffix(name, ".ts"):
|
||||
for _, m := range tsRead.FindAllStringSubmatch(string(text), -1) {
|
||||
read(reads, m[1], rel)
|
||||
}
|
||||
case name != "module.json":
|
||||
// A file the module ships: a NAME= line is a word an env-file gives.
|
||||
for _, m := range envLine.FindAllStringSubmatch(string(text), -1) {
|
||||
set[m[1]] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range goFiles {
|
||||
text, _ := os.ReadFile(p)
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
for _, m := range goRead.FindAllStringSubmatch(string(text), -1) {
|
||||
word := m[1]
|
||||
if word == "" {
|
||||
word = consts[filepath.Dir(p)][m[2]]
|
||||
}
|
||||
if word != "" {
|
||||
read(reads, word, filepath.ToSlash(rel))
|
||||
}
|
||||
}
|
||||
}
|
||||
module := filepath.Base(dir)
|
||||
for word, files := range reads {
|
||||
if set[word] || Given[word] != "" {
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string][]string{}
|
||||
}
|
||||
for f := range files {
|
||||
out[module][word] = append(out[module][word], f)
|
||||
}
|
||||
sort.Strings(out[module][word])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func read(reads map[string]map[string]bool, word, file string) {
|
||||
if reads[word] == nil {
|
||||
reads[word] = map[string]bool{}
|
||||
}
|
||||
reads[word][file] = true
|
||||
}
|
||||
|
||||
// setBy collects the words a manifest sets: every key of an `env` object and of an `environment`'s `variables`.
|
||||
func setBy(v any, set map[string]bool) {
|
||||
switch v := v.(type) {
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
if m, ok := e.(map[string]any); ok && (k == "env" || k == "variables") {
|
||||
for word := range m {
|
||||
set[word] = true
|
||||
}
|
||||
}
|
||||
setBy(e, set)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
setBy(e, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Listed is the list's `<module> <WORD>` lines; a line starting with # and a blank line are not entries.
|
||||
func Listed(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.Join(strings.Fields(s.Text()), " ")
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// Judge answers the reads not on the list (refused) and the list's lines no read stands for (stale).
|
||||
func Judge(found map[string]map[string][]string, listed map[string]bool) (refused, stale []string) {
|
||||
seen := map[string]bool{}
|
||||
for module, words := range found {
|
||||
for word := range words {
|
||||
line := module + " " + word
|
||||
seen[line] = true
|
||||
if !listed[line] {
|
||||
refused = append(refused, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
for line := range listed {
|
||||
if !seen[line] {
|
||||
stale = append(stale, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(refused)
|
||||
sort.Strings(stale)
|
||||
return refused, stale
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
module, word, _ := strings.Cut(line, " ")
|
||||
fmt.Printf("%s reads %s (%s), which its manifest does not set and the mesh gives no process: set it in the "+
|
||||
"manifest, or read what the mesh gives\n", module, word, strings.Join(found[module][word], ", "))
|
||||
}
|
||||
for _, line := range stale {
|
||||
fmt.Printf("checks/environment/%s names %q, which no code reads unset any more: take it off the list\n", listFile, line)
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("environment: every MESH_* word a module reads is set by its manifest or given by the mesh (%d still listed)\n", len(listed))
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// firstCount is how many lines the list held when it was first written: it only shrinks.
|
||||
const firstCount = 63
|
||||
|
||||
func module(t *testing.T, root, name, manifest string, files map[string]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
for path, text := range map[string]string{"module.json": manifest} {
|
||||
files[path] = text
|
||||
}
|
||||
for path, text := range files {
|
||||
full := filepath.Join(dir, filepath.FromSlash(path))
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A word read and set by nothing is found, in Go (a literal, a constant of the same directory) and in TypeScript; one
|
||||
// the manifest sets (an env, an environment's variables, an env-file line it ships) or the mesh gives every process is
|
||||
// not; tests, node_modules and dist are not read.
|
||||
func TestAWordReadThatNothingSetsIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+
|
||||
`"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{
|
||||
"cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`,
|
||||
"cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`,
|
||||
"cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`,
|
||||
"cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`,
|
||||
"client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`,
|
||||
"files/env": "MESH_FROM_FILE=${config:x}\n",
|
||||
"index.ts": `const f = env.MESH_FROM_FILE;`,
|
||||
"test/x.test.ts": `env.MESH_IN_A_TEST_DIR`,
|
||||
"node_modules/x.ts": `env.MESH_VENDORED`,
|
||||
})
|
||||
module(t, root, "quiet", `{"module":"quiet"}`, map[string]string{"main.go": `package main; var _ = os.Getenv("MESH_NODE")`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]map[string][]string{"reader": {
|
||||
"MESH_REACH": {"cmd/a/main.go"},
|
||||
"MESH_LITERAL": {"cmd/a/other.go"},
|
||||
"MESH_TS": {"client.ts"},
|
||||
"MESH_BRACKET": {"client.ts"},
|
||||
}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true})
|
||||
if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) ||
|
||||
!reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) {
|
||||
t.Errorf("refused %v, stale %v", refused, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue itself: every word a module reads is set or given, but the list's, and the list only shrinks.
|
||||
func TestTheCatalogueReadsOnlyWordsTheMeshSets(t *testing.T) {
|
||||
found, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
t.Errorf("%s: read, set by nothing, and not on checks/environment/unset", line)
|
||||
}
|
||||
for _, line := range stale {
|
||||
t.Errorf("checks/environment/unset names %q, which no code reads unset any more", line)
|
||||
}
|
||||
if len(listed) > firstCount {
|
||||
t.Errorf("checks/environment/unset holds %d lines, more than its first %d: it only shrinks", len(listed), firstCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and the code and files of its module.
|
||||
modules
|
||||
@@ -0,0 +1,67 @@
|
||||
# The MESH_* words a module's code still reads that its manifest does not set and the mesh gives no process,
|
||||
# `<module> <WORD>` a line, as checks/environment first found them (issue 1966). Each is a contract nobody keeps: fix
|
||||
# it (the manifest sets it, or the code reads what the mesh gives) and take its line off. A line no code reads unset any
|
||||
# more is refused, so this list only shrinks; a new read set by nothing is refused, never added here.
|
||||
baserow MESH_BASEROW_EMAIL
|
||||
baserow MESH_BASEROW_HOST
|
||||
baserow MESH_BASEROW_PASSWORD
|
||||
claude-code MESH_AGENT_ACCOUNT
|
||||
claude-code MESH_AGENT_HOME
|
||||
claude-licence-manager MESH_ANTHROPIC_TOKEN_ENDPOINT
|
||||
claude-licence-manager MESH_ANTHROPIC_USAGE_ENDPOINT
|
||||
confluence MESH_CONFLUENCE_EMAIL
|
||||
confluence MESH_CONFLUENCE_TOKEN
|
||||
confluence MESH_CONFLUENCE_URL
|
||||
disk-load MESH_DISK_MEASURE_EVERY
|
||||
gitlab MESH_GITLAB_TOKEN
|
||||
gitlab MESH_GITLAB_URL
|
||||
grafana MESH_GRAFANA_PASSWORD
|
||||
grafana MESH_GRAFANA_TOKEN
|
||||
grafana MESH_GRAFANA_USER
|
||||
home-assistant MESH_HOMEASSISTANT_TOKEN
|
||||
home-assistant MESH_HOMEASSISTANT_WAIT_SECONDS
|
||||
home-assistant MESH_HOMEASSISTANT_WATCH
|
||||
icecast MESH_ICECAST_ADMIN_PASSWORD
|
||||
icecast MESH_ICECAST_ADMIN_USER
|
||||
influxdb MESH_INFLUXDB_ORG
|
||||
influxdb MESH_INFLUXDB_TOKEN
|
||||
jira MESH_JIRA_EMAIL
|
||||
jira MESH_JIRA_TOKEN
|
||||
jira MESH_JIRA_URL
|
||||
letta MESH_LETTA_PASSWORD
|
||||
mailu MESH_MAILU_API_KEY
|
||||
mailu MESH_MAILU_DOMAIN
|
||||
mesh-cli MESH_CONSOLE
|
||||
mesh-cli MESH_VERB
|
||||
mesh-files MESH_OWNED_PATHS
|
||||
mesh-files MESH_SECRET_PLACES
|
||||
mesh-reads MESH_SECRET_PLACES
|
||||
minio MESH_MINIO_ROOT_PASSWORD
|
||||
mongodb MESH_MONGODB_AUTHSOURCE
|
||||
mongodb MESH_MONGODB_HOST
|
||||
mongodb MESH_MONGODB_PASSWORD
|
||||
mongodb MESH_MONGODB_PORT
|
||||
mongodb MESH_MONGODB_USER
|
||||
mosquitto MESH_MQTT_ADMIN_USER
|
||||
mosquitto MESH_MQTT_BROKER_GID
|
||||
mosquitto MESH_MQTT_BROKER_UID
|
||||
mosquitto MESH_MQTT_HOST
|
||||
mosquitto MESH_MQTT_PASSWORD
|
||||
mosquitto MESH_MQTT_PORT
|
||||
mssql MESH_MSSQL_HOST
|
||||
mssql MESH_MSSQL_PASSWORD
|
||||
mssql MESH_MSSQL_PORT
|
||||
mssql MESH_MSSQL_READER_PASSWORD
|
||||
mssql MESH_MSSQL_USER
|
||||
nextcloud MESH_NEXTCLOUD_ADMIN_PASSWORD
|
||||
nextcloud MESH_NEXTCLOUD_CONTAINER
|
||||
nodered MESH_NODERED_TOKEN
|
||||
nodered MESH_NODERED_WAIT_SECONDS
|
||||
redis MESH_REDIS_HOST
|
||||
redis MESH_REDIS_PASSWORD
|
||||
redis MESH_REDIS_PORT
|
||||
redis MESH_REDIS_USERNAME
|
||||
restic MESH_BACKUP_HOUR
|
||||
unifi MESH_UNIFI_PASSWORD
|
||||
unifi MESH_UNIFI_SITE
|
||||
unifi MESH_UNIFI_USERNAME
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package failurekit refuses a build module whose tests do not run mesh-sdk's conformance kit (issue 1901, its thorough
|
||||
// Package failurekit refuses a build module whose tests do not run the build runner's conformance kit (issue 1901, its thorough
|
||||
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
|
||||
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
@@ -71,7 +71,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !runs {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
"(mesh-build-go buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
}
|
||||
}
|
||||
// Every build module keeps steps, so every one runs the left-running case, test step or not (issue 1890).
|
||||
@@ -81,7 +81,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !sweeps {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
}
|
||||
j.Refused = strings.Join(refused, "; ")
|
||||
out = append(out, j)
|
||||
@@ -121,7 +121,7 @@ func runsTheKit(dir, call string) (bool, error) {
|
||||
return err
|
||||
}
|
||||
found = strings.Contains(string(raw), call) &&
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"`)
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance"`)
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// cases is refused, and a module that claims no node-build-* seat is not judged.
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/buildstep/conformance\"\n\n"
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance\"\n\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
|
||||
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
|
||||
for path, body := range map[string]string{
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
// Package gitignored refuses an ignore pattern that hides a module's source (catalogue review F1, and its kind): the
|
||||
// bare pattern `mesh-build-node` in a module's .gitignore, meant for the binary `go build` leaves, also matched the
|
||||
// directory cmd/mesh-build-node, so git ignored every new file there and publish.go was never committed. A path under
|
||||
// a module's cmd/ or internal/ is source: no .gitignore, the module's or the catalogue's, may ignore one, tracked or
|
||||
// new. A binary is ignored by an anchored pattern (`/mesh-build-node`, `/cmd/mesh-build-node/mesh-build-node`).
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Probe is the file name asked of each source directory: a Go file a change might add there.
|
||||
const Probe = "added_by_a_change.go"
|
||||
|
||||
// Hidden is one source path an ignore pattern hides, and the pattern (its file, line and text, as git names it).
|
||||
type Hidden struct {
|
||||
Path, Pattern string
|
||||
}
|
||||
|
||||
// Sources are the source directories of root's modules, relative to root: every directory under modules/*/cmd and
|
||||
// modules/*/internal (node_modules left out).
|
||||
func Sources(root string) ([]string, error) {
|
||||
mods, err := filepath.Glob(filepath.Join(root, "modules", "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, m := range mods {
|
||||
for _, top := range []string{"cmd", "internal"} {
|
||||
base := filepath.Join(m, top)
|
||||
if st, err := os.Stat(base); err != nil || !st.IsDir() {
|
||||
continue
|
||||
}
|
||||
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isSource is whether a file is source a module's build reads: Go and TypeScript files and a Go module's own files.
|
||||
// What a build or a test script leaves beside them (a binary, compiled JavaScript) may be ignored.
|
||||
func isSource(name string) bool {
|
||||
switch filepath.Ext(name) {
|
||||
case ".go", ".ts", ".tsx":
|
||||
return true
|
||||
}
|
||||
return name == "go.mod" || name == "go.sum"
|
||||
}
|
||||
|
||||
// Find asks git which of the source paths root's ignore files hide: each source directory, a Go file added in it, and
|
||||
// every source file there (isSource). It reads the .gitignore files of root's tree through a repository of its own, so it judges a git
|
||||
// checkout and a plain folder alike, and no one's global ignores count.
|
||||
func Find(root string, dirs []string) ([]Hidden, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
var ask bytes.Buffer
|
||||
for _, d := range dirs {
|
||||
ask.WriteString(d + "/\x00" + d + "/" + Probe + "\x00")
|
||||
entries, err := os.ReadDir(filepath.Join(root, d))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && isSource(e.Name()) {
|
||||
ask.WriteString(d + "/" + e.Name() + "\x00")
|
||||
}
|
||||
}
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = &ask
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
|
||||
return nil, nil // nothing is ignored
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -z answers source, line, pattern and path, each ended by NUL.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var hidden []Hidden
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
continue // a negation keeps the path
|
||||
}
|
||||
hidden = append(hidden, Hidden{Path: f[i+3], Pattern: fmt.Sprintf("%s:%s: %s", f[i], f[i+1], f[i+2])})
|
||||
}
|
||||
return hidden, nil
|
||||
}
|
||||
|
||||
// Programs are the paths `go build` writes a compiled program to, for every main package of every Go module under
|
||||
// root (issue 1577, met twice in one milestone: a module's program committed with `git add -A`): `go build` in the
|
||||
// package's directory writes it there, and `go build ./<dir>` from the module's root writes it at the root, each named
|
||||
// after the package's import path. Relative to root, sorted; directories named node_modules, testdata or vendor, and
|
||||
// .git, are left out, and so is a path that is a directory of the tree.
|
||||
func Programs(root string) ([]string, error) {
|
||||
var mods []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && skipped(d.Name()) && p != root {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "go.mod" {
|
||||
mods = append(mods, filepath.Dir(p))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range mods {
|
||||
modPath, err := modulePath(filepath.Join(m, "go.mod"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = filepath.WalkDir(m, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if p != m {
|
||||
if skipped(d.Name()) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(p, "go.mod")); err == nil {
|
||||
return filepath.SkipDir // a module of its own, walked on its own
|
||||
}
|
||||
}
|
||||
isMain, err := mainPackage(p)
|
||||
if err != nil || !isMain {
|
||||
return err
|
||||
}
|
||||
name := filepath.Base(p)
|
||||
if p == m {
|
||||
name = pathBase(modPath)
|
||||
}
|
||||
for _, dir := range []string{p, m} {
|
||||
out := filepath.Join(dir, name)
|
||||
if st, err := os.Stat(out); err == nil && st.IsDir() {
|
||||
continue // go build refuses to write over a directory, so no program lands there
|
||||
}
|
||||
rel, _ := filepath.Rel(root, out)
|
||||
seen[filepath.ToSlash(rel)] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for p := range seen {
|
||||
out = append(out, p)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Unignored names each of programs (paths from root) that root's ignore files do not ignore: a program a local build
|
||||
// leaves there that `git add -A` would commit.
|
||||
func Unignored(root string, programs []string) ([]string, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-n", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = strings.NewReader(strings.Join(programs, "\x00") + "\x00")
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); !(err == nil || ok && ee.ExitCode() == 1) {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -n -z answers source, line, pattern and path for every path asked; the source is empty for one no pattern
|
||||
// matches, and a pattern starting with ! keeps the path.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var open []string
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
open = append(open, f[i+3])
|
||||
}
|
||||
}
|
||||
return open, nil
|
||||
}
|
||||
|
||||
func skipped(name string) bool {
|
||||
switch name {
|
||||
case ".git", "node_modules", "testdata", "vendor":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// modulePath reads the module line of a go.mod.
|
||||
func modulePath(goMod string) (string, error) {
|
||||
b, err := os.ReadFile(goMod)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "module" {
|
||||
return strings.Trim(f[1], `"`), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s names no module", goMod)
|
||||
}
|
||||
|
||||
// pathBase is the name `go build` gives a program at a module's root: its path's last element, a major version
|
||||
// suffix (/v2) left out.
|
||||
func pathBase(modPath string) string {
|
||||
parts := strings.Split(modPath, "/")
|
||||
last := parts[len(parts)-1]
|
||||
if len(parts) > 1 && len(last) > 1 && last[0] == 'v' && strings.Trim(last[1:], "0123456789") == "" {
|
||||
last = parts[len(parts)-2]
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
// mainPackage is whether dir holds a Go file, not a test, of package main.
|
||||
func mainPackage(dir string) (bool, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
n := e.Name()
|
||||
if e.IsDir() || filepath.Ext(n) != ".go" || strings.HasSuffix(n, "_test.go") {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, n))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "package" {
|
||||
if f[1] == "main" {
|
||||
return true, nil
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The kind of catalogue review F1: a bare pattern hides the command directory it shares a name with; an anchored one
|
||||
// hides the binary alone; a glob that ignores source is refused too; a negation keeps a path.
|
||||
func TestFindNamesEachPatternThatHidesSource(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
hidden []string
|
||||
}{
|
||||
"a bare name hides cmd/<name>": {"nfs-server\n", []string{"modules/m/cmd/nfs-server/", "modules/m/cmd/nfs-server/" + Probe, "modules/m/cmd/nfs-server/main.go"}},
|
||||
"anchored to the binary": {"/nfs-server\n/cmd/nfs-server/nfs-server\n", nil},
|
||||
"a glob ignoring Go source": {"*.go\n!main.go\n", []string{"modules/m/cmd/" + Probe, "modules/m/cmd/nfs-server/" + Probe,
|
||||
"modules/m/internal/" + Probe, "modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"a pattern over internal/": {"internal/\n", []string{"modules/m/internal/", "modules/m/internal/" + Probe, "modules/m/internal/x/",
|
||||
"modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"nothing but generated JavaScript": {"*.js\n", nil},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range map[string]string{
|
||||
"modules/m/.gitignore": c.ignore,
|
||||
"modules/m/cmd/nfs-server/main.go": "package main\n",
|
||||
"modules/m/cmd/nfs-server/run.js": "",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/node_modules/y/index.go": "",
|
||||
"modules/other/cmd/other/other.go": "package main\n",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paths []string
|
||||
for _, h := range got {
|
||||
paths = append(paths, h.Path)
|
||||
if !strings.HasPrefix(h.Pattern, "modules/m/.gitignore:") {
|
||||
t.Errorf("%s: %s is named hidden by %q, want the module's .gitignore", name, h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
if strings.Join(paths, ",") != strings.Join(c.hidden, ",") {
|
||||
t.Errorf("%s: hidden %v, want %v", name, paths, c.hidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: no ignore pattern hides a path under a module's cmd/ or internal/.
|
||||
func TestNoIgnorePatternHidesAModulesSource(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no module source directory found: the check reads nothing")
|
||||
}
|
||||
hidden, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hidden {
|
||||
t.Errorf("%s is hidden by %s: a module's source is never ignored; anchor the pattern to the binary (/<name>, /cmd/<name>/<name>)", h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
|
||||
// The kind of issue 1577, met twice in one milestone: a module's program `go build` left in its folder was committed.
|
||||
// Every program a main package builds, in its own directory and at its module's root, is found, and one no ignore file
|
||||
// names is open; a directory of the same name is no program's place.
|
||||
func TestEveryProgramABuildLeavesIsIgnored(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"modules/m/go.mod": "module example.com/m/v2\n",
|
||||
"modules/m/main.go": "// a program\npackage main\n",
|
||||
"modules/m/cmd/m-tools/main.go": "package main\n",
|
||||
"modules/m/cmd/m-tools/main_test.go": "package main\n",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/internal/x/x_test.go": "package main\n",
|
||||
"modules/m/node_modules/y/y.go": "package main\n",
|
||||
"modules/m/tool/go.mod": "module tool\n",
|
||||
"modules/m/tool/cmd/tool/main.go": "package main\n",
|
||||
"modules/m/tool/tool/source.go": "package tool\n",
|
||||
"checks/c/go.mod": "module example.com/checks/c\n",
|
||||
"checks/c/main.go": "package main\n",
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
open []string
|
||||
}{
|
||||
"nothing ignored": {"", []string{"checks/c/c", "modules/m/cmd/m-tools/m-tools", "modules/m/m", "modules/m/m-tools",
|
||||
"modules/m/tool/cmd/tool/tool"}},
|
||||
"each anchored": {"/checks/c/c\n/modules/m/cmd/m-tools/m-tools\n/modules/m/m\n/modules/m/m-tools\n/modules/m/tool/cmd/tool/tool\n", nil},
|
||||
"one kept by a negation": {"/checks/c/c\n/modules/m/*\n!/modules/m/m\n/modules/m/cmd/m-tools/m-tools\n/modules/m/tool/cmd/tool/tool\n",
|
||||
[]string{"modules/m/m"}},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range files {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, ".gitignore"), []byte(c.ignore), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
programs, err := Programs(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "checks/c/c,modules/m/cmd/m-tools/m-tools,modules/m/m,modules/m/m-tools,modules/m/tool/cmd/tool/tool"
|
||||
if got := strings.Join(programs, ","); got != want {
|
||||
t.Fatalf("%s: programs %s, want %s", name, got, want)
|
||||
}
|
||||
open, err := Unignored(root, programs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(open, ",") != strings.Join(c.open, ",") {
|
||||
t.Errorf("%s: open %v, want %v", name, open, c.open)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: every program a local build leaves is ignored, so `git add -A` never takes one.
|
||||
func TestNoProgramABuildLeavesIsCommitted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
programs, err := Programs(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(programs) == 0 {
|
||||
t.Fatal("no main package found: the check reads nothing")
|
||||
}
|
||||
open, err := Unignored(root, programs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range open {
|
||||
t.Errorf("%s is the program a local go build leaves, and no ignore file names it: add /%s to the catalogue's .gitignore", p, p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gitignored
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gofmt
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,58 @@
|
||||
// Package gofmt refuses a Go file of the catalogue that gofmt would change (the second of a kind in one milestone:
|
||||
// buildstep's generated.go and claude-code's config_test.go reached a join unformatted). A check here holds every
|
||||
// module to it, where a module's own tests hold only themselves.
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/format"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Files are the Go files under root, relative to it: every .go file but those in .git, node_modules and a testdata
|
||||
// directory (a test's fixture may be anything).
|
||||
func Files(root string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case ".git", "node_modules", "testdata":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(d.Name(), ".go") && d.Type().IsRegular() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Unformatted are the files gofmt would change, and why for one that does not parse.
|
||||
func Unformatted(root string, files []string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, f := range files {
|
||||
src, err := os.ReadFile(filepath.Join(root, f))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
got, err := format.Source(src)
|
||||
switch {
|
||||
case err != nil:
|
||||
out[f] = "it does not parse: " + err.Error()
|
||||
case !bytes.Equal(got, src):
|
||||
out[f] = "gofmt changes it: run gofmt -w " + f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnformattedNamesWhatGofmtChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for f, src := range map[string]string{
|
||||
"modules/a/a.go": "package a\n",
|
||||
"modules/b/b.go": "package b\nimport (\n\t\"os\"\n\t\"fmt\"\n)\nvar _, _ = fmt.Sprint, os.Exit\n",
|
||||
"modules/c/testdata/c.go": "package c",
|
||||
"modules/d/node_modules/d.go": "package d",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
if err := os.WriteFile(p, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 || len(got) != 1 || got["modules/b/b.go"] == "" {
|
||||
t.Errorf("files %v, unformatted %v; want a.go and b.go read, b.go named", files, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own Go files are as gofmt writes them.
|
||||
func TestTheCataloguesGoIsFormatted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no Go file found: the check reads nothing")
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f, why := range got {
|
||||
t.Errorf("%s: %s", f, why)
|
||||
}
|
||||
}
|
||||
@@ -2,4 +2,4 @@ module checks/health
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -2,4 +2,4 @@ module checks/manifests
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package manifests reads every module's manifest as the catalogue's tree holds it and refuses what no judge could
|
||||
// read: a module.json that does not decode, and two manifests that name one module.
|
||||
// read: a module.json that does not decode, a module.json giving one field twice, and two manifests that name one module.
|
||||
//
|
||||
// It judges no manifest (novox/hq ADR 0578 §2): the checker judges every change's manifests with the indexer's judge, the
|
||||
// one judge, so the field, seat, bus and catalogue rules are said once, there. Until 2026-11 this check parsed each
|
||||
@@ -9,6 +9,8 @@
|
||||
package manifests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -18,7 +20,9 @@ import (
|
||||
)
|
||||
|
||||
// Problems is what is wrong with the tree's manifests under root/modules, one line each: a module.json whose bytes are
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), then a module named by two manifests.
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), one giving a field twice (a decoder keeps the
|
||||
// last and drops the first without a word: a merge of two branches that each added the field writes it so), then a module
|
||||
// named by two manifests.
|
||||
func Problems(root string) ([]string, error) {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
@@ -38,6 +42,10 @@ func Problems(root string) ([]string, error) {
|
||||
out = append(out, fmt.Sprintf("%s: %v", rel, err))
|
||||
continue
|
||||
}
|
||||
if twice := givenTwice(raw); twice != "" {
|
||||
out = append(out, fmt.Sprintf("%s: the field %s is given twice; a judge reads only the last", rel, twice))
|
||||
continue
|
||||
}
|
||||
if named[m.Module] {
|
||||
out = append(out, fmt.Sprintf("%s: %s was already given; two manifests name one module", rel, m.Module))
|
||||
continue
|
||||
@@ -46,3 +54,47 @@ func Problems(root string) ([]string, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// givenTwice is the first field an object of raw gives twice, as its path (`invokes`, `build.artifacts[0].env`), or empty.
|
||||
func givenTwice(raw []byte) string {
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
var walk func(path string) string
|
||||
walk = func(path string) string {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch t {
|
||||
case json.Delim('{'):
|
||||
seen := map[string]bool{}
|
||||
for d.More() {
|
||||
k, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
key, _ := k.(string)
|
||||
at := key
|
||||
if path != "" {
|
||||
at = path + "." + key
|
||||
}
|
||||
if seen[key] {
|
||||
return at
|
||||
}
|
||||
seen[key] = true
|
||||
if twice := walk(at); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
case json.Delim('['):
|
||||
for i := 0; d.More(); i++ {
|
||||
if twice := walk(fmt.Sprintf("%s[%d]", path, i)); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk("")
|
||||
}
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The tree's manifests are ones a judge can read, and no module is named twice. A row is a catalogue of its own: a
|
||||
// manifest that is no JSON and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// manifest that is no JSON, one giving a field twice and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// check does not know is not (it judges nothing: the checker does, ADR 0578 §2). The catalogue itself passes.
|
||||
func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
@@ -19,6 +21,10 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
{name: "a manifest the judge can read", files: map[string]string{"a": `{"module":"a","version":"1"}`}},
|
||||
{name: "a field no part knows is the judge's to refuse", files: map[string]string{"a": `{"module":"a","version":"1","unheard":true}`}},
|
||||
{name: "bytes that are no manifest", files: map[string]string{"a": `{"module":`}, refused: "modules/a/module.json"},
|
||||
{name: "a field given twice", files: map[string]string{"a": `{"module":"a","version":"1","invokes":["x.y"],"build":{"artifacts":[{}]},"invokes":["z.w"]}`},
|
||||
refused: "modules/a/module.json: the field invokes is given twice"},
|
||||
{name: "a field given twice deeper", files: map[string]string{"a": `{"module":"a","version":"1","build":{"artifacts":[{"name":"t","name":"u"}]}}`},
|
||||
refused: "the field build.artifacts[0].name is given twice"},
|
||||
{name: "two manifests of one module", files: map[string]string{"a": `{"module":"x","version":"1"}`, "b": `{"module":"x","version":"1"}`},
|
||||
refused: "two manifests name one module"},
|
||||
} {
|
||||
@@ -48,3 +54,45 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
t.Fatalf("the catalogue holds manifests no judge can read:\n%s", strings.Join(real, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// No manifest of this catalogue claims the seat mesh-catalog: the indexer of the delivery holds it, and the module that
|
||||
// held it before has dropped its claim (ADR 0604 §6). A row is a tree of its own: a manifest claiming it is named.
|
||||
func TestNoManifestClaimsTheSeatMeshCatalog(t *testing.T) {
|
||||
claimants := func(root string) []string {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue // Problems names it
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "mesh-catalog" {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
row := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(row, "modules", "a"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(row, "modules", "a", "module.json"),
|
||||
[]byte(`{"module":"a","version":"1","claims":[{"name":"mesh-catalog","scope":"mesh"}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := claimants(row); len(got) != 1 || got[0] != "a" {
|
||||
t.Fatalf("a manifest claiming the seat is named, got %v", got)
|
||||
}
|
||||
if got := claimants(filepath.Join("..", "..")); len(got) != 0 {
|
||||
t.Errorf("the catalogue's manifests %v claim the seat mesh-catalog: the indexer of the delivery holds it", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Command noplacement refuses what the repository path of placement left behind (novox/hq ADR 0567 §10, §12).
|
||||
//
|
||||
// Placement, which module runs on which node, is bus state held by the seat mesh-placement; no file of the
|
||||
// Placement, which module runs on which node, is bus state held by the seat mesh-module-assigner; no file of the
|
||||
// catalogue says it, and no manifest reads it from the delivery indexer, which no longer keeps it. It refuses:
|
||||
//
|
||||
// 1. a `placement` directory (or file) at the root of the catalogue;
|
||||
@@ -8,7 +8,8 @@
|
||||
// 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16);
|
||||
//
|
||||
// 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…`
|
||||
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`;
|
||||
// or `mesh-delivery-indexer_placement…`), or the assigner's retired state placement (`mesh-delivery-assigner.placement…`,
|
||||
// `mesh-module-assigner.placement…`, its views and its read tool), in any string value, such as `reads`;
|
||||
//
|
||||
// The controller's assignments, which item 4 once listed readers of, are refused with every other name of the
|
||||
// controller by checks/controllerdeps (novox/hq ADR 0590 §5).
|
||||
@@ -27,14 +28,17 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Prefixes are the names of the indexer's placement parts; a manifest string starting with one names them.
|
||||
var Prefixes = []string{"mesh-delivery-indexer.placement", "mesh-delivery-indexer_placement"}
|
||||
// Prefixes are the names of the indexer's placement parts and the retired names of the assigner's state, now
|
||||
// assignments (novox/hq ADR 0641); a manifest string starting with one names them.
|
||||
var Prefixes = []string{"mesh-delivery-indexer.placement", "mesh-delivery-indexer_placement",
|
||||
"mesh-delivery-assigner.placement", "mesh-delivery-assigner_placement", "mesh-delivery-assigner.views.placement",
|
||||
"mesh-delivery-assigner.assigner_read_placement", "mesh-module-assigner.placement"}
|
||||
|
||||
// Findings is every refusal in the catalogue at root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
var out []string
|
||||
if _, err := os.Lstat(filepath.Join(root, "placement")); err == nil {
|
||||
out = append(out, "placement: the repository holds no placement; it is bus state of the seat mesh-placement (novox/hq ADR 0567 §10)")
|
||||
out = append(out, "placement: the repository holds no placement; it is bus state of the seat mesh-module-assigner (novox/hq ADR 0567 §10)")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "particulars")); err == nil {
|
||||
out = append(out, "particulars: the repository lists no node names; the checker hands them to the check (novox/hq ADR 0567 §16)")
|
||||
@@ -55,7 +59,7 @@ func Findings(root string) ([]string, error) {
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(file))
|
||||
for _, s := range named(doc, nil) {
|
||||
out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s))
|
||||
out = append(out, fmt.Sprintf("%s: names %q, a name that is retired: the indexer keeps no placement (novox/hq ADR 0567 §12) and the seat mesh-module-assigner's state is assignments (novox/hq ADR 0641)", module, s))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
@@ -99,5 +103,5 @@ func main() {
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no placement directory, and no manifest reads the indexer's placement")
|
||||
fmt.Println("no placement directory, and no manifest names the retired placement")
|
||||
}
|
||||
|
||||
@@ -13,13 +13,17 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
|
||||
files map[string]string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""},
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.assignments"]}`}, ""},
|
||||
{"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"},
|
||||
{"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"},
|
||||
{"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`},
|
||||
{"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"},
|
||||
{"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"},
|
||||
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""},
|
||||
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-module-assigner.assignments"]}`}, ""},
|
||||
{"the assigner's retired state", map[string]string{"modules/e/module.json": `{"module":"e","reads":["mesh-delivery-assigner.placement"]}`}, `e: names "mesh-delivery-assigner.placement"`},
|
||||
{"its retired views family", map[string]string{"modules/f/module.json": `{"module":"f","consumes":["mesh-delivery-assigner.views.placement"]}`}, "views.placement"},
|
||||
{"its retired read tool", map[string]string{"modules/g/module.json": `{"module":"g","invokes":["mesh-delivery-assigner.assigner_read_placement"]}`}, "assigner_read_placement"},
|
||||
{"the seat's retired verb", map[string]string{"modules/h/module.json": `{"module":"h","invokes":["mesh-module-assigner.placement"]}`}, "mesh-module-assigner.placement"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
+1
-1
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/owned
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+1
-1
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/paths
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/predefined
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+10
-1
@@ -2,4 +2,13 @@ module checks/readable
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require (
|
||||
git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../../modules/disk-load/diskreadings
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ../../modules/restic/backupgood
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 // indirect
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// the owner's manifest went with clause 5, and announcetest holds a module to its own list, not to its readers.
|
||||
//
|
||||
// An owner announces from its code. A Go module announces when its code imports mesh-sdk's go/announce: every
|
||||
// announce.State literal there is read, its Name a string, a constant of the package, or a constant of mesh-sdk this
|
||||
// announce.State literal there is read, its Name a string, a constant of the package, or a constant of an owner's package this
|
||||
// check knows (Known). A TypeScript module announces when its code says the event `states` or calls the SDK's keeps:
|
||||
// every `{ name: "...", readable: true }` there is read. An owner that announces nothing from code is read as before
|
||||
// on the mesh, and is not judged; neither is an owner outside the catalogue (another repository's).
|
||||
@@ -29,13 +29,15 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
|
||||
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
|
||||
)
|
||||
|
||||
// Known are the mesh-sdk constants an announcement may name a state by, by the selector the code writes.
|
||||
// Known are the constants of the owners' published packages an announcement may name a state by, by the selector the
|
||||
// code writes: disk-load's readings and restic's last good night (ADR 0634 §4: a reading's layout is its owner's).
|
||||
var Known = map[string]string{
|
||||
"health.GoodState": health.GoodState,
|
||||
"health.ReadingsState": health.ReadingsState,
|
||||
"backupgood.GoodState": backupgood.GoodState,
|
||||
"diskreadings.ReadingsState": diskreadings.ReadingsState,
|
||||
}
|
||||
|
||||
const announcePkg = `"git.novox.be/novox/mesh-sdk/go/announce"`
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestFindings(t *testing.T) {
|
||||
{"go, a known mesh-sdk name and readers", map[string]string{
|
||||
"modules/r/module.json": `{"module":"r","reads":["o.good","o.s"]}`,
|
||||
"modules/o/module.json": `{"module":"o"}`,
|
||||
"modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: health.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""},
|
||||
"modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: backupgood.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""},
|
||||
{"go, a name that cannot be read", map[string]string{
|
||||
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
|
||||
"modules/o/module.json": `{"module":"o"}`,
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/removedpackages
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,166 @@
|
||||
// Command removedpackages refuses a Go source of the catalogue that imports what mesh-sdk gave up in the release that
|
||||
// narrowed it (novox/hq ADR 0593 §1): a package that left for its owner, and the build runner's names that left
|
||||
// mesh-sdk's buildstep for mesh-build-go's package.
|
||||
//
|
||||
// A module compiles against the mesh-sdk version it pins, so a source that still names a removed package builds until the
|
||||
// pin moves, and then breaks in a delivery of its own; and a later change could bring the import back with the old pin.
|
||||
// This check refuses it at once, from the source alone:
|
||||
//
|
||||
// - an import of a package in Removed (or one under it);
|
||||
// - in a file importing mesh-sdk's buildstep (under any name), a selector naming anything that is not in
|
||||
// BuildstepContract: what is left of that package is what the delivery parts read, and the runner is
|
||||
// mesh-build-go's package (git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep).
|
||||
//
|
||||
// It reads the non-test and test Go files under modules/ and checks/ (testdata, vendor and node_modules left out).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding. The lists are the release's: a package that goes later is added here with the change.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const sdk = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// Removed are the packages of mesh-sdk that left it, each with where its logic is now.
|
||||
var Removed = map[string]string{
|
||||
"buildcpu": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/buildcpu",
|
||||
"buildstep/conformance": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance",
|
||||
"delivery/buildrun": "the builder's package (mesh-delivery-builder), imported by its repository's modules",
|
||||
"compose": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/frombus": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/fromruntime": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/judge": "the assigner's own package",
|
||||
"takein": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"validate": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"facts": "nothing: the controller they served is ended (ADR 0590)",
|
||||
"example": "nothing: no module imports it (ADR 0593 §12)",
|
||||
"packageguard": "nothing: a test helper of mesh-sdk's own",
|
||||
}
|
||||
|
||||
// BuildstepContract are the names mesh-sdk's buildstep keeps: the failure contract, the fetch request, the stale-step sweep,
|
||||
// the step id and the module tree shape the delivery parts read.
|
||||
var BuildstepContract = map[string]bool{
|
||||
"AskWhat": true, "Boot": true, "By": true, "CapacitySetting": true, "Clip": true, "ConsoleFailure": true, "CutLine": true,
|
||||
"EndOne": true, "EndStale": true, "EndStaleAtStart": true, "EndStaleRound": true, "Failure": true, "FetchRequest": true,
|
||||
"FetchRequestProperties": true, "Hidden": true, "Launchers": true, "LauncherSuffix": true, "LineBytes": true,
|
||||
"ModuleTree": true, "NewID": true, "OutputBytes": true, "Owner": true, "PackagesArg": true, "ParseFetchRequest": true,
|
||||
"PassingLine": true, "RestartReason": true, "Scan": true, "ScanOf": true, "StaleAttempts": true, "StaleRetry": true,
|
||||
"StaleServed": true,
|
||||
}
|
||||
|
||||
// Findings is every refusal in the Go sources under root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
var out []string
|
||||
for _, top := range []string{"modules", "checks"} {
|
||||
err := filepath.WalkDir(filepath.Join(root, top), func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case "testdata", "vendor", "node_modules", ".mesh-build":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
found, err := fileFindings(path, filepath.ToSlash(rel))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, found...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fileFindings(path, rel string) ([]string, error) {
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", rel, err)
|
||||
}
|
||||
var out []string
|
||||
buildstep := ""
|
||||
for _, imp := range file.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sub, ok := strings.CutPrefix(p, sdk+"/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for pkg, to := range Removed {
|
||||
if sub == pkg || strings.HasPrefix(sub, pkg+"/") {
|
||||
out = append(out, fmt.Sprintf("%s: imports %s, which left mesh-sdk; it is %s (novox/hq ADR 0593 §1)", rel, p, to))
|
||||
break
|
||||
}
|
||||
}
|
||||
if sub == "buildstep" {
|
||||
buildstep = "buildstep"
|
||||
if imp.Name != nil {
|
||||
buildstep = imp.Name.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
if buildstep == "" || buildstep == "_" || buildstep == "." {
|
||||
return out, nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
ast.Inspect(file, func(n ast.Node) bool {
|
||||
sel, ok := n.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
if !ok || id.Name != buildstep || !sel.Sel.IsExported() || BuildstepContract[sel.Sel.Name] || seen[sel.Sel.Name] {
|
||||
return true
|
||||
}
|
||||
seen[sel.Sel.Name] = true
|
||||
out = append(out, fmt.Sprintf("%s: uses %s.%s, which is the build runner's, no longer mesh-sdk's buildstep; import git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep (novox/hq ADR 0593 §1)", rel, buildstep, sel.Sel.Name))
|
||||
return true
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: removedpackages <catalogue root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no source imports a package mesh-sdk gave up")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0593 §1: a source that imports a package mesh-sdk gave up, or names the build runner through mesh-sdk's buildstep, is
|
||||
// refused naming where it went; the contract that stays, the runner's own package and a testdata copy are not.
|
||||
func TestFindingsRefuseWhatMeshSDKGaveUp(t *testing.T) {
|
||||
const head = "package x\n\nimport (\n"
|
||||
for _, row := range []struct {
|
||||
name, file, src string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/delivery\"\n)\n\nvar _ = delivery.StepsState\n", ""},
|
||||
{"a removed package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n\nvar _ = compose.Resolve\n", "imports git.novox.be/novox/mesh-sdk/go/compose, which left mesh-sdk; it is the node-engine"},
|
||||
{"under a removed package", "modules/a/x_test.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose/frombus\"\n)\n", "the node-engine"},
|
||||
{"buildcpu", "checks/c/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildcpu\"\n)\n", "buildstep/buildcpu"},
|
||||
{"the runner through mesh-sdk's buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", "uses buildstep.RunAsk, which is the build runner's"},
|
||||
{"the runner through an alias", "modules/a/x.go", head + "\tsdkstep \"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = sdkstep.Puller{}\n", "uses sdkstep.Puller"},
|
||||
{"the contract of buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.NewID\nvar _ buildstep.Failure\n", ""},
|
||||
{"the runner's own package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", ""},
|
||||
{"a copy in testdata", "modules/a/testdata/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n", ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := filepath.Join(root, row.file)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(row.src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The script's run of this check over the real tree (novox/hq ADR 0529 clause 2).
|
||||
func TestTheCatalogueImportsNothingMeshSDKGaveUp(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# every Go source of the modules and of the checks, whose imports are read.
|
||||
modules
|
||||
checks
|
||||
@@ -36,13 +36,7 @@ import (
|
||||
// composition is every function or table the check lets stay in mesh-sdk, as "path-from-go/:name", and why. It names no
|
||||
// manifest judgement: each is the composer's refusal of composition itself (ADR 0578 clause 5).
|
||||
var composition = map[string]string{
|
||||
"catalogue/consumers.go:consumersPlaceholderProblems": "composition: rendering a consumer's template",
|
||||
"catalogue/collections.go:collectedPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/environment_into.go:placeholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/seat_contributions.go:seatPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
"catalogue/setting_takes.go:takenSettingsProblems": "composition: whether stored settings still compose",
|
||||
"catalogue/template.go:HeldTemplateProblem": "composition: the take-in refusal of a resolved build",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
}
|
||||
|
||||
// Problems is what mesh-sdk's Go tree under root (its go/ directory) holds of a manifest judge, one line each, but for
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/secrets
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/templates
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestCheckAddressesRefusesTheControllersAddresses(t *testing.T) {
|
||||
{"`mesh-controller.<verb>`", true},
|
||||
{"the mesh-controller module is gone.", false},
|
||||
{"located in mesh-controller#246", false},
|
||||
{"ask mesh-placement.placement", false},
|
||||
{"ask mesh-module-assigner.assignments", false},
|
||||
} {
|
||||
got := checkAddresses([]text{{file: "m/a.go", line: 1, s: c.s}})
|
||||
if (len(got) == 1) != c.want {
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
58
|
||||
56
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: checks/health fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -2,10 +2,10 @@ module adwaita
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -6,38 +6,34 @@ directory and the file the mesh writes there from its setting `older-store-names
|
||||
|
||||
## Why a module beside the store
|
||||
|
||||
The store is the `distribution` module. A module that provides the artifact store cannot also build an
|
||||
artifact: building publishes to the store, so the store would be needed to create itself, and the
|
||||
controller's module check refuses that manifest. So the store's tools are a second module, as the
|
||||
controller's refusal says to do. **Assign it to the machine that holds the store**: its tools read the
|
||||
store's files through the store's own container, and anywhere else they answer that the container is not
|
||||
on that machine.
|
||||
The store is the `distribution` module, the registry seats' holder. These tools became a module of their own when the
|
||||
indexer's judge refused a store holder that builds an artifact; that rule is gone (issue 1966), and the sweep stays its
|
||||
own module, holding the seat `mesh-artifact-sweep`. It requires no provision: what it holds, releases and removes it
|
||||
asks the registry seats `oci-artifact-registry` and `container-image-registry` (see *The sweep through the registry
|
||||
seats*), and what the store holds it reads through the same seats' `list` (novox/hq ADR 0632 §4): it reads neither the
|
||||
store's files nor its read side, so it runs on any machine.
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | | what |
|
||||
|---|---|---|
|
||||
| `artifact_store_repositories` | r | every repository: tags and what each names, how many manifests (tagged or not), size |
|
||||
| `artifact_store_usage` | r | the store's size, the largest repositories, bytes shared between repositories, bytes no manifest marks (what the nightly collector frees next) |
|
||||
| `artifact_store_usage` | r | the store's size, each manifest's as its seat says it, and the largest repositories |
|
||||
| `artifact_store_references` | r | what the controller's records and the catalogue's definitions say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_collect` | a | a dry run unless `dry_run` is false: what the controller would let go of, and the bytes the nightly collector would free then and now. A real run needs `why` and asks the controller's `collect` |
|
||||
|
||||
### How the store is read
|
||||
|
||||
The store's door lists repositories and tags, but not a manifest no tag names, and the mesh pins every
|
||||
machine by digest, so that is most of them. So the bundle reads the store's own files through its own
|
||||
container (`docker exec mesh-registry`, busybox `find`, `stat` and `cat`), and only reads them: every
|
||||
blob with its size, every manifest each repository holds, what each tag names, and each manifest's
|
||||
content, four hundred to an exec. What it read it keeps (a manifest is named by its content, so it never
|
||||
changes), and the next call reads only what is new. A manifest that could not be read is counted and
|
||||
said; what it marks beyond itself is then unknown, so sizes may read low and freed bytes high, and the
|
||||
answer says so.
|
||||
The store's read side lists repositories and tags, but not a manifest no tag names, and the mesh pins every
|
||||
machine by digest, so that is most of them. So the bundle asks the two registry seats' `list`
|
||||
(`container-image-registry` for images and indexes, `oci-artifact-registry` for bundles and archives), which the
|
||||
store's holder answers from the store's own files: each manifest's digest, whether it is an index and the manifests it
|
||||
names, the tags naming it, its size and whether a hold keeps it (novox/hq ADR 0632 §4).
|
||||
|
||||
docker runs as the tool runner's account; a socket that refuses it is asked again through `sudo -n`,
|
||||
never with a prompt, as the container runtime's own tools do.
|
||||
|
||||
A manifest *marks* its own content, its configuration and its layers, and an index marks the manifests
|
||||
it lists. The store's collector removes every blob no manifest marks, so these numbers are its own.
|
||||
A manifest's size is its own content and every blob it names, as its seat says it, so a layer two manifests share
|
||||
is counted in each, and the bytes the store's nightly collector frees are the sizes of the manifests that go, an
|
||||
upper bound. The address the bundle says the store is at comes from `${seat:container-image-registry:reach}`, the
|
||||
image seat holder's listen (ADR 0632 §1), and is used for the catalogue's definitions only, never to reach the store.
|
||||
|
||||
### The states of a manifest
|
||||
|
||||
@@ -61,7 +57,7 @@ record names leaves its platform manifests unrecorded with it.
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
definition the catalogue holds (the seat mesh-catalog's verbs `modules`, then `module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
@@ -99,19 +95,27 @@ manifest that stays and would lose a platform it names — to be read before a r
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Against a fake container that prints what the two scripts would: the listing parsed (a repository name
|
||||
with slashes, a cut listing refused), manifests read with one absent or unreadable said as unread, a
|
||||
digest never becoming a path outside the blobs, the mark (an index, a shared blob, a blob nothing marks),
|
||||
every state, what the records keep that the store lacks, bytes freed counting a blob shared with a kept
|
||||
manifest as kept, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
Against a fake of the two registry seats' `list`: the store read through it (an index and the manifests it names, a
|
||||
bundle's holder, a hold, the seat's sizes), every state, what the records keep that the store lacks, bytes freed as the
|
||||
sizes of what goes, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
the fall-back when the references let go of cannot be carried, a manifest a definition pins under the
|
||||
store's current or an older name, a definition not read said first, escalation through `sudo -n`, and that the
|
||||
store's current or an older name, a definition not read said first, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs and the
|
||||
catalogue's two tools.
|
||||
Any command other than the two read-only scripts fails the test.
|
||||
Any ask other than a seat's `list` fails the read tools' tests.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
### The sweep through the registry seats
|
||||
|
||||
The sweep writes nothing to the store's door (CAT-AS1, ADR 0617 §2): it asks the registry seats the store's holder
|
||||
serves. Each kept bundle or archive is held on `oci-artifact-registry` and each kept copy of an upstream base on
|
||||
`container-image-registry` (`hold`, with why it is kept) before anything goes; each eligible one is released, then
|
||||
removed for `retention` on its seat (`release`, `remove`). It reads a reference in both forms, `artifact-store://<path>`
|
||||
and `<seat>/<module>/<artifact>@<digest>`, as one artifact, and says artifacts-gone in the form the build named it. A
|
||||
call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no
|
||||
seat refuses clears it (`sweepseats.go`).
|
||||
|
||||
@@ -23,10 +23,11 @@ import (
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the catalog's tools these definitions are read through.
|
||||
// CatalogueModules and CatalogueModule are the verbs of the seat mesh-catalog these definitions are read through
|
||||
// (ADR 0604 §3); the answers are catalogue.CatalogModules and catalogue.CatalogOne.
|
||||
const (
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
CatalogueModules = "seat:mesh-catalog.modules"
|
||||
CatalogueModule = "seat:mesh-catalog.module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
@@ -279,11 +280,8 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
// The verb takes no commit: it answers the trunk's module, and the answer names the commit it is at.
|
||||
raw, err := c.Ask(CatalogueModule, map[string]any{"module": m.Module})
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
@@ -293,6 +291,7 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Commit string `json:"commit"`
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
@@ -301,9 +300,11 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
if m.Commit != "" {
|
||||
// Kept by the commit the answer names, which may be newer than the list's: a definition and its commit never
|
||||
// change together.
|
||||
if at := firstOf(one.Module.Commit, m.Commit); at != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[key] = pins
|
||||
c.kept[m.Module+"@"+at+"|"+nameKey] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
|
||||
@@ -152,7 +152,7 @@ func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
// references is one artifact_store_references tool, kept across calls as the runtime keeps the bundle.
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
@@ -211,7 +211,7 @@ func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
t.Errorf("asked the seat's module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,41 +1,57 @@
|
||||
package main
|
||||
|
||||
// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
|
||||
// The artifact store, through its seats, and the bus, as the holder reaches them through this module's runtime.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// door is registry.Store at the store's address as this machine reaches it.
|
||||
type door struct{ s registry.Store }
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
|
||||
func (d *door) Address() string { return d.s.Address }
|
||||
func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
|
||||
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
|
||||
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
|
||||
return d.s.IsIndex(ctx, r)
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite responses (skip one and
|
||||
// go on, abandon the sweep for the other), and collapsing them into "an error" is how a cautious loop became one that did
|
||||
// nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// door is the store as its seats' list answers it (ADR 0632 §4), at the address this machine reaches it by: it reads
|
||||
// only; what the sweep holds and removes it asks the registry seats too (sweepseats.go).
|
||||
type door struct {
|
||||
address string
|
||||
store *Store
|
||||
}
|
||||
func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
|
||||
func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
|
||||
spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
|
||||
if err == nil {
|
||||
d.s.Spare = spare
|
||||
}
|
||||
|
||||
func (d *door) Address() string { return d.address }
|
||||
|
||||
// Holds is whether the store holds what a reference names: an image's manifest, a bundle's or an archive's blob. Gone when
|
||||
// its seat lists none (ADR 0526 clause 3: a registration by hand is refused at bytes that are not there).
|
||||
func (d *door) Holds(_ context.Context, r string) error {
|
||||
_, err := d.store.listed(r)
|
||||
return err
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
|
||||
// reach, each time, so a store that moved is reached where it is now.
|
||||
func DoorAt(file string) func() (Door, error) {
|
||||
// IsIndex is whether a reference is an index the store holds, as its seat lists it. Gone when it does not hold it.
|
||||
func (d *door) IsIndex(_ context.Context, r string) (bool, error) {
|
||||
e, err := d.store.listed(r)
|
||||
return e.Index, err
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat container-image-registry's
|
||||
// reach, each time, so a store that moved is said where it is now; the store is read through s.
|
||||
func DoorAt(file string, s *Store) func() (Door, error) {
|
||||
return func() (Door, error) {
|
||||
if file == "" {
|
||||
return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
|
||||
@@ -48,15 +64,27 @@ func DoorAt(file string) func() (Door, error) {
|
||||
if address == "" {
|
||||
return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
|
||||
}
|
||||
return &door{s: registry.Store{Address: address}}, nil
|
||||
return &door{address: address, store: s}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
|
||||
type runtimeBus struct{}
|
||||
type runtimeBus struct{ views *views.Reader }
|
||||
|
||||
func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
// followed says whether the state is the indexer's or the assigner's, which the holder asks once and follows by events
|
||||
// (ADR 0615) rather than reads.
|
||||
func (b runtimeBus) followed(state string) bool { return b.views != nil && b.views.Follows(state) }
|
||||
|
||||
func (b runtimeBus) Keys(state string) ([]string, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Keys(state)
|
||||
}
|
||||
return stdio.State(state).Keys()
|
||||
}
|
||||
func (b runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Get(state, key)
|
||||
}
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
@@ -90,7 +118,7 @@ func (runtimeBus) Publish(subject string, body any) error {
|
||||
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps and
|
||||
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
|
||||
func follow(h *Holder) {
|
||||
func follow(h *Holder, v *views.Reader) {
|
||||
<-stdio.Served()
|
||||
watch := func(state string, handle func(stdio.StateChange) error) {
|
||||
for {
|
||||
@@ -119,11 +147,22 @@ func follow(h *Holder) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
watch(WantsRead, func(c stdio.StateChange) error {
|
||||
if !c.Current {
|
||||
go observe()
|
||||
// The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns
|
||||
// is the first view, not a change.
|
||||
var current atomic.Bool
|
||||
current.Store(true)
|
||||
for {
|
||||
err := v.Watch(WantsRead, func(string, []byte, bool) {
|
||||
if !current.Load() {
|
||||
go observe()
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
return nil
|
||||
})
|
||||
h.logf("watching %s: %v; asked again in 10s", WantsRead, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
current.Store(false)
|
||||
observe()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
)
|
||||
|
||||
// The holders whose states this module reads, one constant each (a reader names its holder once; ADR 0634 §4).
|
||||
const (
|
||||
assignerModule = "mesh-delivery-assigner"
|
||||
indexerModule = "mesh-delivery-indexer"
|
||||
)
|
||||
|
||||
// The seat mesh-artifact-sweep types the answer of its verb held and the body of its event artifacts-gone; this module
|
||||
// holds the seat, so it answers and says them in those types.
|
||||
type (
|
||||
// HeldAnswer is the answer of the seat's verb held.
|
||||
HeldAnswer = sweep.HeldAnswer
|
||||
// ArtifactsGone is the body of the seat's event artifacts-gone.
|
||||
ArtifactsGone = sweep.ArtifactsGone
|
||||
)
|
||||
|
||||
// GoneSubject is where the seat says artifacts-gone.
|
||||
const GoneSubject = sweep.SubjectArtifactsGone
|
||||
|
||||
// EncodeArtifactsGone is the event's body, the references sorted; it refuses one naming no module, no fingerprint or no
|
||||
// gone reference.
|
||||
func EncodeArtifactsGone(e ArtifactsGone) ([]byte, error) {
|
||||
e.Event = sweep.EventArtifactsGone
|
||||
e.Gone = append([]string(nil), e.Gone...)
|
||||
sort.Strings(e.Gone)
|
||||
if e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 {
|
||||
return nil, fmt.Errorf("%s names a module, a fingerprint and at least one gone reference", sweep.EventArtifactsGone)
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
@@ -18,33 +18,37 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// What the holder reads and keeps.
|
||||
const (
|
||||
WantsRead = delivery.AssignerModule + ".wants"
|
||||
BuildsRead = delivery.BuilderModule + ".builds"
|
||||
WantsRead = assignerModule + ".wants"
|
||||
BuildsRead = builds.BuilderModule + ".builds"
|
||||
HavesRead = "node-engine.have"
|
||||
// ManifestsRead is the indexer's manifests: the images a manifest pins by itself.
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
// FindingsState is the holder's own findings (health.FindingsState): image-missing.
|
||||
FindingsState = health.FindingsState
|
||||
ManifestsRead = indexerModule + "." + catalog.StateManifests
|
||||
// FindingsState is the holder's own findings (meshconditions.StateFindings): image-missing.
|
||||
FindingsState = meshconditions.StateFindings
|
||||
// Module is this module, the observer its findings name.
|
||||
Module = "artifact-store-tools"
|
||||
// GoneSubject is where the seat says artifacts-gone.
|
||||
GoneSubject = "mesh.seat." + catalogue.ArtifactSweepSeat + ".event." + delivery.ArtifactsGoneEvent
|
||||
)
|
||||
|
||||
// The sweep's bounds, the controller's (cmd/mesh-controller/collect.go mostPerSweep, sweepBudget).
|
||||
@@ -69,26 +73,25 @@ type Bus interface {
|
||||
Publish(subject string, body any) error
|
||||
}
|
||||
|
||||
// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store.
|
||||
// Door is the artifact store as its seats' list answers it, read only, behind an interface so a test needs no store. What
|
||||
// the sweep holds and removes it asks the registry seats (Seats, sweepseats.go).
|
||||
type Door interface {
|
||||
Address() string
|
||||
Hold(ctx context.Context, reference string) (bool, error)
|
||||
Holds(ctx context.Context, reference string) error
|
||||
IsIndex(ctx context.Context, reference string) (bool, error)
|
||||
LetGo(ctx context.Context, reference string) error
|
||||
// SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257).
|
||||
SpareKept(ctx context.Context, keptImages, touching []string) error
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
type Holder struct {
|
||||
Bus Bus
|
||||
Door func() (Door, error)
|
||||
// Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
// Listed is the store as its seats list it now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
Listed func(ctx context.Context) (*View, error)
|
||||
Acts handacts.Recorder
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
Acts meshhandactsclient.Recorder
|
||||
// Seats asks a registry seat's verb, `seat:<seat>.<verb>` (hold, remove): the runtime's stdio.Ask in main.
|
||||
Seats func(key string, body any) (json.RawMessage, error)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
|
||||
sweep sync.Mutex // one sweep at a time
|
||||
mu sync.Mutex
|
||||
@@ -99,7 +102,7 @@ type Holder struct {
|
||||
type Build struct {
|
||||
Module, FP, Key string
|
||||
At time.Time
|
||||
Artifacts []delivery.BuiltArtifact
|
||||
Artifacts []builds.BuiltArtifact
|
||||
// Manifest is the digest of the manifest it built, in the indexer's `manifests`.
|
||||
Manifest string
|
||||
}
|
||||
@@ -112,7 +115,7 @@ func (h *Holder) Builds() ([]Build, error) {
|
||||
}
|
||||
var out []Build
|
||||
for _, k := range keys {
|
||||
module, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
module, fp, err := builds.ParseBuilderBuildKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -123,8 +126,8 @@ func (h *Holder) Builds() ([]Build, error) {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State != delivery.BuildDone {
|
||||
b, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State != builds.BuildDone {
|
||||
continue
|
||||
}
|
||||
at, err := h.endOf(k)
|
||||
@@ -149,7 +152,7 @@ func (h *Holder) endOf(key string) (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("the history of the builder's build %s: %w", key, err)
|
||||
}
|
||||
for _, r := range revs {
|
||||
if b, err := delivery.DecodeBuilderBuild(r.Value); err == nil && b.State == delivery.BuildDone && r.At.After(at) {
|
||||
if b, err := builds.DecodeBuilderBuild(r.Value); err == nil && b.State == builds.BuildDone && r.At.After(at) {
|
||||
at = r.At
|
||||
}
|
||||
}
|
||||
@@ -181,11 +184,11 @@ func (h *Holder) Wants() ([]Want, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the want %s: %w", k, err)
|
||||
}
|
||||
var w delivery.Want
|
||||
if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.FP == "" {
|
||||
var w meshwants.WantsByWantNodeModuleValue
|
||||
if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.Fp == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Want{Node: node, Module: module, FP: w.FP})
|
||||
out = append(out, Want{Node: node, Module: module, FP: w.Fp})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -217,8 +220,8 @@ func KeepSet(builds []Build, wants []Want) map[string]*Ref {
|
||||
out := map[string]*Ref{}
|
||||
for _, b := range builds {
|
||||
for _, a := range b.Artifacts {
|
||||
reference := catalogue.Recorded(a.Reference)
|
||||
if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" {
|
||||
reference := canonical(registryForms.Recorded(a.Reference))
|
||||
if _, ours := registryForms.InArtifactStore(reference); !ours || a.Kind == "package" {
|
||||
continue
|
||||
}
|
||||
r := out[reference]
|
||||
@@ -352,17 +355,17 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
|
||||
// against is what each build the builder's log says ended stood on, by build key: the bases' references.
|
||||
func (h *Holder) against() (map[string][]string, error) {
|
||||
entries, err := h.Bus.LogSince(delivery.BuildLogRead, h.now().Add(-delivery.BuilderLogDays*24*time.Hour))
|
||||
entries, err := h.Bus.LogSince(builds.BuildLogRead, h.now().Add(-builds.BuilderLogDays*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]string{}
|
||||
for _, raw := range entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(raw.Value)
|
||||
if err != nil || e.Event != delivery.BuildEndedEvent {
|
||||
e, err := builds.DecodeBuildLogEntry(raw.Value)
|
||||
if err != nil || e.Event != builds.BuildEndedEvent {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.BuilderBuildKey(e.Module, e.FP)
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -383,7 +386,7 @@ func Mirrors(v *View, builds []Build, kept map[string][]string, against map[stri
|
||||
continue
|
||||
}
|
||||
for d := range v.L.Repos[name].Revisions {
|
||||
r := catalogue.ArtifactStoreScheme + name + "@" + d
|
||||
r := artifactref.ArtifactStoreScheme + name + "@" + d
|
||||
out[r] = &Ref{Reference: r, Kind: "image"}
|
||||
}
|
||||
}
|
||||
@@ -397,7 +400,7 @@ func Mirrors(v *View, builds []Build, kept map[string][]string, against map[stri
|
||||
continue
|
||||
}
|
||||
for _, base := range bases {
|
||||
if m := out[catalogue.Recorded(base)]; m != nil {
|
||||
if m := out[registryForms.Recorded(base)]; m != nil {
|
||||
m.Builds = append(m.Builds, b)
|
||||
why := kept[b.Key]
|
||||
if len(why) > 0 {
|
||||
@@ -445,7 +448,7 @@ func (j *judged) records(repository string, collected bool) *Records {
|
||||
}
|
||||
}
|
||||
r.Counts[state]++
|
||||
if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, catalogue.ArtifactStoreScheme), repository)) ||
|
||||
if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, artifactref.ArtifactStoreScheme), repository)) ||
|
||||
(state == "collected" && !collected) {
|
||||
continue
|
||||
}
|
||||
@@ -511,13 +514,13 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
return nil, err
|
||||
}
|
||||
a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}}
|
||||
var keptArchives, keptImages, eligible []string
|
||||
var keptArchives, eligible []string
|
||||
for _, r := range j.sorted() {
|
||||
switch {
|
||||
case r.Kept && r.Kind == "archive":
|
||||
case r.Kept && (r.Kind == "archive" || isCopy(r.Reference)):
|
||||
// What retention would remove unless held: a bundle or an archive, and a copied base (ADR 0617 §2).
|
||||
keptArchives = append(keptArchives, r.Reference)
|
||||
case r.Kept:
|
||||
keptImages = append(keptImages, r.Reference)
|
||||
default:
|
||||
if held, known := inStore(j.view, r.Reference); known && !held {
|
||||
continue // collected already
|
||||
@@ -543,33 +546,25 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
a.Store = door.Address()
|
||||
within, stop := context.WithTimeout(ctx, SweepBudget)
|
||||
defer stop()
|
||||
// Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete.
|
||||
// Hold before letting go (ADR 0251 §8, ADR 0617 §2): each kept archive and copied base is held on its seat; one that
|
||||
// cannot be held stops the sweep before any removal and writes the sweep's finding.
|
||||
var missing []string
|
||||
for _, r := range keptArchives {
|
||||
wrote, err := door.Hold(within, r)
|
||||
switch {
|
||||
case err == nil:
|
||||
if wrote {
|
||||
a.HoldersWritten++
|
||||
}
|
||||
case errors.Is(err, registry.Gone):
|
||||
if held, known := inStore(j.view, r); known && !held {
|
||||
missing = append(missing, r)
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if _, err := h.holdOnSeat(r, keptWhy(j.refs[r])); err != nil {
|
||||
a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err)
|
||||
a.Left = len(eligible)
|
||||
h.sweepFinding("the hold of "+r, err)
|
||||
return a, nil
|
||||
}
|
||||
a.HoldersWritten++
|
||||
}
|
||||
a.Held, a.Missing = len(keptArchives)-len(missing), len(missing)
|
||||
h.sayGone(j, missing, "a kept archive the store does not hold")
|
||||
if s.Confirmed {
|
||||
if err := door.SpareKept(within, keptImages, eligible); err != nil {
|
||||
a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
||||
a.Left = len(eligible)
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
refused := false
|
||||
for i, r := range eligible {
|
||||
if i >= s.Most || within.Err() != nil {
|
||||
a.Left = len(eligible) - i
|
||||
@@ -583,7 +578,7 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
if !s.Confirmed {
|
||||
// An index waits for a confirmed collect (ADR 0251 §7, ADR 0257 §4).
|
||||
index, err := door.IsIndex(within, r)
|
||||
if err != nil && !errors.Is(err, registry.Gone) && !errors.Is(err, registry.ErrNotOurs) {
|
||||
if err != nil && !errors.Is(err, Gone) && !errors.Is(err, ErrNotOurs) {
|
||||
a.Stopped = fmt.Sprintf("the store could not say whether %s is an index, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
break
|
||||
@@ -593,37 +588,35 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
continue
|
||||
}
|
||||
}
|
||||
err := door.LetGo(within, r)
|
||||
switch {
|
||||
case err == nil || errors.Is(err, registry.Gone):
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
a.Skipped++
|
||||
continue
|
||||
default:
|
||||
a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
}
|
||||
if a.Stopped != "" {
|
||||
break
|
||||
}
|
||||
// Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A").
|
||||
// Removed through its seat (ADR 0617 §2), recorded in the hand-act log, each one (ADR 0565 §10; the operator's
|
||||
// "option A").
|
||||
why := s.Why
|
||||
if why == "" {
|
||||
why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds"
|
||||
}
|
||||
if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why,
|
||||
if err := h.removeOnSeat(r, why); err != nil {
|
||||
a.Stopped = fmt.Sprintf("a registry seat refused to remove %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
h.sweepFinding("the removal of "+r, err)
|
||||
refused = true
|
||||
break
|
||||
}
|
||||
if _, err := h.Acts.Record(meshhandactsclient.Request{What: "let go of " + r + " (" + s.By + ")", Why: why,
|
||||
Cause: "artifact-store-sweep", Verb: "collect"}); err != nil {
|
||||
h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err)
|
||||
}
|
||||
a.LetGo = append(a.LetGo, r)
|
||||
}
|
||||
h.sayGone(j, a.LetGo, "let go of")
|
||||
if !refused {
|
||||
h.sweepClean()
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// sayGone says artifacts-gone for each build that named one of refs: the builder builds a key again whose artifact is gone.
|
||||
func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
byKey := map[string]*delivery.ArtifactsGone{}
|
||||
byKey := map[string]*ArtifactsGone{}
|
||||
var keys []string
|
||||
for _, r := range refs {
|
||||
ref := j.refs[r]
|
||||
@@ -633,16 +626,17 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
for _, b := range ref.Builds {
|
||||
g := byKey[b.Key]
|
||||
if g == nil {
|
||||
g = &delivery.ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: h.now().UTC()}
|
||||
at := h.now().UTC()
|
||||
g = &ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: &at}
|
||||
byKey[b.Key] = g
|
||||
keys = append(keys, b.Key)
|
||||
}
|
||||
g.Gone = append(g.Gone, r)
|
||||
g.Gone = append(g.Gone, namedBy(b, r))
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := delivery.EncodeArtifactsGone(*byKey[k])
|
||||
raw, err := EncodeArtifactsGone(*byKey[k])
|
||||
if err == nil {
|
||||
err = h.Bus.Publish(GoneSubject, json.RawMessage(raw))
|
||||
}
|
||||
@@ -652,6 +646,17 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
}
|
||||
}
|
||||
|
||||
// namedBy is the reference b names r by, in the form its key holds (the older one or the seat's): the builder matches
|
||||
// artifacts-gone against its own key.
|
||||
func namedBy(b Build, r string) string {
|
||||
for _, a := range b.Artifacts {
|
||||
if rec := registryForms.Recorded(a.Reference); canonical(rec) == r {
|
||||
return rec
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// AfterBuild is the sweep at a build's end.
|
||||
func (h *Holder) AfterBuild(key string) {
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
|
||||
@@ -665,30 +670,30 @@ func (h *Holder) AfterBuild(key string) {
|
||||
}
|
||||
|
||||
// Held is the seat's `held`: each reference, held, gone or unknown with why. Holds and writes nothing.
|
||||
func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) {
|
||||
func (h *Holder) Held(refs []string) (HeldAnswer, error) {
|
||||
if len(refs) == 0 {
|
||||
return delivery.HeldAnswer{}, errors.New("held judges the references given, and none was")
|
||||
return HeldAnswer{}, errors.New("held judges the references given, and none was")
|
||||
}
|
||||
if len(refs) > 64 {
|
||||
return delivery.HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs))
|
||||
return HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs))
|
||||
}
|
||||
for _, r := range refs {
|
||||
if _, ours := catalogue.InArtifactStore(r); !ours {
|
||||
return delivery.HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
|
||||
if _, ours := registryForms.InArtifactStore(r); !ours {
|
||||
return HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
|
||||
}
|
||||
}
|
||||
door, err := h.Door()
|
||||
if err != nil {
|
||||
return delivery.HeldAnswer{}, err
|
||||
return HeldAnswer{}, err
|
||||
}
|
||||
asking, stop := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer stop()
|
||||
a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}}
|
||||
a := HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}}
|
||||
for _, r := range refs {
|
||||
switch err := door.Holds(asking, r); {
|
||||
switch err := door.Holds(asking, canonical(r)); {
|
||||
case err == nil:
|
||||
a.Held = append(a.Held, r)
|
||||
case errors.Is(err, registry.Gone):
|
||||
case errors.Is(err, Gone):
|
||||
a.Gone = append(a.Gone, r)
|
||||
default:
|
||||
if a.Unknown == nil {
|
||||
@@ -724,7 +729,7 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
}
|
||||
byImage := map[string]*Image{}
|
||||
add := func(module, fp, in string) error {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
key, err := builds.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -732,14 +737,14 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
b, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
images := []string{}
|
||||
for _, a := range b.Artifacts {
|
||||
if (a.Kind == "image" || a.Kind == "upstream") && a.Reference != "" {
|
||||
images = append(images, catalogue.Routed(catalogue.Recorded(a.Reference), address))
|
||||
images = append(images, registryForms.Routed(registryForms.Recorded(a.Reference), address))
|
||||
}
|
||||
}
|
||||
pinned, err := h.Pinned(b.Manifest)
|
||||
@@ -773,7 +778,7 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
return out, fmt.Errorf("the haves: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
n, module, isNode, err := delivery.ParseHaveKey(k)
|
||||
n, module, isNode, err := engineclient.ParseHaveKey(k)
|
||||
if err != nil || isNode || n != node {
|
||||
continue
|
||||
}
|
||||
@@ -781,12 +786,12 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var have delivery.Have
|
||||
if !found || json.Unmarshal(raw, &have) != nil || have.FP == "" {
|
||||
var have nodeengine.HaveByNodeModuleValue
|
||||
if !found || json.Unmarshal(raw, &have) != nil || have.Fp == "" {
|
||||
continue
|
||||
}
|
||||
out.SentKnown = true
|
||||
if err := add(module, have.FP, "sent"); err != nil {
|
||||
if err := add(module, have.Fp, "sent"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
@@ -805,7 +810,7 @@ func (h *Holder) Pinned(manifest string) ([]string, error) {
|
||||
if manifest == "" {
|
||||
return nil, nil
|
||||
}
|
||||
key, err := delivery.ManifestsKey(manifest)
|
||||
key, err := catalogclient.ManifestsKey(manifest)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -813,14 +818,14 @@ func (h *Holder) Pinned(manifest string) ([]string, error) {
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
m, err := catalogclient.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
var parsed struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if json.Unmarshal([]byte(m.JSON), &parsed) != nil {
|
||||
if json.Unmarshal([]byte(m.Manifest), &parsed) != nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out []string
|
||||
@@ -840,8 +845,8 @@ func inStoreAt(image, address string) string {
|
||||
if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") {
|
||||
return ""
|
||||
}
|
||||
r := catalogue.Recorded(image)
|
||||
if _, ours := catalogue.InArtifactStore(r); !ours {
|
||||
r := registryForms.Recorded(image)
|
||||
if _, ours := registryForms.InArtifactStore(r); !ours {
|
||||
return ""
|
||||
}
|
||||
return r
|
||||
@@ -865,7 +870,7 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
for _, b := range j.built {
|
||||
builds[b.Module+" "+b.FP] = b
|
||||
}
|
||||
now := map[string]health.Finding{}
|
||||
now := map[string]meshconditions.FindingsByKeyValue{}
|
||||
for _, w := range j.wants {
|
||||
b, ok := builds[w.Module+" "+w.FP]
|
||||
if !ok {
|
||||
@@ -874,7 +879,7 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
var refs []string
|
||||
for _, a := range b.Artifacts {
|
||||
if a.Kind == "image" {
|
||||
refs = append(refs, catalogue.Recorded(a.Reference))
|
||||
refs = append(refs, registryForms.Recorded(a.Reference))
|
||||
}
|
||||
}
|
||||
// And what its manifest pins by itself under the store's address, as the controller judged S25.
|
||||
@@ -899,7 +904,16 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
return fmt.Errorf("the findings: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if k == sweepFindingKey {
|
||||
continue // the sweep's own finding, which the sweep clears
|
||||
}
|
||||
if _, still := now[k]; !still {
|
||||
// A finding of a first view unanswered is the reader's (views.Reader), cleared by its answer, not by a sweep.
|
||||
if raw, found, err := h.Bus.Get(FindingsState, k); err == nil && found {
|
||||
if f, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && f.Kind == views.UnansweredKind {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := h.Bus.Delete(FindingsState, k); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -911,11 +925,11 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if was, err := health.DecodeFinding(raw); err == nil && was.Summary == f.Summary {
|
||||
if was, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && was.Summary == f.Summary {
|
||||
continue // written only when what it says changes
|
||||
}
|
||||
}
|
||||
if _, err := health.EncodeFinding(f); err != nil {
|
||||
if err := meshconditionsclient.CheckFinding(f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.Bus.Put(FindingsState, k, f); err != nil {
|
||||
@@ -926,11 +940,11 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages).
|
||||
func imageMissing(node, module, image string, at time.Time) health.Finding {
|
||||
func imageMissing(node, module, image string, at time.Time) meshconditions.FindingsByKeyValue {
|
||||
id := module + "." + node
|
||||
return health.Finding{V: health.FindingVersion, Key: health.FindingKey(health.ScopeModule, id, ImageMissing), Kind: ImageMissing,
|
||||
Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node},
|
||||
Severity: health.Warning, Resolver: health.ResolverOperator,
|
||||
return meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: meshconditionsclient.Key(meshconditionsclient.ScopeModule, id, ImageMissing), Kind: ImageMissing,
|
||||
Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: id, Machine: node},
|
||||
Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+
|
||||
"not hold (the evidence names it); the running container is untouched", module, node),
|
||||
Headline: capital(module) + " is not updated on " + node,
|
||||
@@ -938,7 +952,7 @@ func imageMissing(node, module, image string, at time.Time) health.Finding {
|
||||
"was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", module, node, node),
|
||||
Needs: fmt.Sprintf("build %s again; the details name the image.", module),
|
||||
Resolved: "Resolved: the store holds the image of " + module + " again",
|
||||
Evidence: []health.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}},
|
||||
Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}},
|
||||
Source: "S25", Observer: Module, At: at.UTC()}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,9 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
@@ -158,7 +159,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
// The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject.
|
||||
var mu sync.Mutex
|
||||
var acts, gone []string
|
||||
if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) {
|
||||
if _, err := nc.Subscribe("mesh.seat."+meshhandacts.Seat+".tool.hand-act", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
acts = append(acts, string(m.Data))
|
||||
mu.Unlock()
|
||||
@@ -173,15 +174,32 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
// The registry seats' holder on the bus (CAT-AS1): the sweep holds and removes through their verbs.
|
||||
for _, seat := range []string{"oci-artifact-registry", "container-image-registry"} {
|
||||
if _, err := nc.Subscribe("mesh.seat."+seat+".tool.*", func(m *nats.Msg) {
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(m.Data, &body)
|
||||
mu.Lock()
|
||||
out, err := door.seat("seat:"+seat+"."+m.Subject[strings.LastIndex(m.Subject, ".")+1:], body)
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
out, _ = json.Marshal(map[string]string{"error": err.Error()})
|
||||
}
|
||||
_ = m.Respond(out)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Data, nil
|
||||
}}
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: ask}
|
||||
|
||||
// A build's end heard from the builds bucket, as follow hands it over.
|
||||
kv, _ := bus.kv(BuildsRead)
|
||||
|
||||
@@ -3,19 +3,28 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
)
|
||||
|
||||
// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
|
||||
@@ -30,7 +39,7 @@ type memBus struct {
|
||||
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
|
||||
var out []stdio.LogEntry
|
||||
for _, e := range b.log {
|
||||
if read == delivery.BuildLogRead && !e.At.Before(at) {
|
||||
if read == builds.BuildLogRead && !e.At.Before(at) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
@@ -85,37 +94,63 @@ func (b *memBus) Publish(subject string, body any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
|
||||
// memDoor is the store's door in memory, read only, and the registry seats' holder: what it holds, which references are
|
||||
// indexes, what the seats were asked (asked: "hold <ref>", "let go <ref>"; calls: each seat call as its key and name), the
|
||||
// holds asked (held), and the calls it refuses (refuse: by "<verb> <ref>").
|
||||
type memDoor struct {
|
||||
holds map[string]bool
|
||||
indexes map[string]bool
|
||||
asked []string
|
||||
calls []string
|
||||
held map[string]bool
|
||||
refuse map[string]error
|
||||
}
|
||||
|
||||
// seat is the registry seats' holder, as Holder.Seats calls it.
|
||||
func (d *memDoor) seat(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
var in struct{ Name, Digest, Reason string }
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
ref := artifactref.ArtifactStoreScheme + in.Name + "@" + in.Digest
|
||||
if seat == "oci-artifact-registry" {
|
||||
ref = artifactref.ArtifactStoreScheme + in.Name + "/blobs/" + in.Digest
|
||||
}
|
||||
d.calls = append(d.calls, key+" "+in.Name+"@"+in.Digest+" "+in.Reason)
|
||||
if err := d.refuse[verb+" "+ref]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.held == nil {
|
||||
d.held = map[string]bool{}
|
||||
}
|
||||
switch verb {
|
||||
case "hold":
|
||||
d.asked = append(d.asked, "hold "+ref)
|
||||
d.held[ref] = true
|
||||
return json.Marshal(map[string]any{"held": true})
|
||||
case "release":
|
||||
released := d.held[ref] // what is not held answers released false, as the seat's definition says
|
||||
delete(d.held, ref)
|
||||
return json.Marshal(map[string]any{"released": released})
|
||||
case "remove":
|
||||
if d.held[ref] {
|
||||
return nil, fmt.Errorf("remove: refused: %s@%s is held; release it first", in.Name, in.Digest)
|
||||
}
|
||||
d.asked = append(d.asked, "let go "+ref)
|
||||
delete(d.holds, ref)
|
||||
return json.Marshal(map[string]any{"removed": true})
|
||||
}
|
||||
return nil, fmt.Errorf("no verb %s", key)
|
||||
}
|
||||
|
||||
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
|
||||
func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
|
||||
d.asked = append(d.asked, "hold "+r)
|
||||
if !d.holds[r] {
|
||||
return false, registry.Gone
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
func (d *memDoor) Holds(_ context.Context, r string) error {
|
||||
if !d.holds[r] {
|
||||
return registry.Gone
|
||||
return Gone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
|
||||
func (d *memDoor) LetGo(_ context.Context, r string) error {
|
||||
d.asked = append(d.asked, "let go "+r)
|
||||
delete(d.holds, r)
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) SpareKept(context.Context, []string, []string) error {
|
||||
d.asked = append(d.asked, "spare")
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirror is a copy of an upstream base in the store, the n-th.
|
||||
func mirror(n int) string {
|
||||
@@ -146,8 +181,8 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}}
|
||||
listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
for n := 1; n <= 7; n++ {
|
||||
key, _ := delivery.BuilderBuildKey("letta", fp(n))
|
||||
raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
|
||||
key, _ := builds.BuilderBuildKey("letta", fp(n))
|
||||
raw, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)},
|
||||
{Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}})
|
||||
@@ -175,7 +210,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
door.holds[mirror(n)] = true
|
||||
}
|
||||
for n := 1; n <= 7; n++ {
|
||||
e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
|
||||
e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
|
||||
Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"}
|
||||
switch n {
|
||||
case 2:
|
||||
@@ -183,21 +218,21 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
case 7:
|
||||
e.Against = []string{mirror(2)}
|
||||
}
|
||||
raw, _ := delivery.EncodeBuildLogEntry(e)
|
||||
raw, _ := builds.EncodeBuildLogEntry(e)
|
||||
b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At})
|
||||
}
|
||||
manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta",
|
||||
JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
|
||||
manifest, _ := catalogclient.EncodeManifest(catalog.ManifestsByDigestValue{Module: "letta",
|
||||
Manifest: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
|
||||
`{"id":"built","type":"container","image":"${artifact:image}"}]}`})
|
||||
mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64))
|
||||
mk, _ := catalogclient.ManifestsKey(strings.Repeat("d", 64))
|
||||
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
|
||||
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
|
||||
want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true})
|
||||
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
|
||||
var acts []string
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
|
||||
Now: func() time.Time { return day.Add(48 * time.Hour) },
|
||||
Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
acts = append(acts, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
@@ -246,18 +281,43 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
left int
|
||||
skipped int
|
||||
unlog int
|
||||
// held is how many kept archives and copies are held (7 when 0); seatForm names letta's 2nd and 3rd builds in the
|
||||
// seat's form; refuse is the seat call refused, "<verb> <ref>".
|
||||
held int
|
||||
seatForm bool
|
||||
refuse string
|
||||
}{
|
||||
{name: "a build's end", how: "build", letGo: all},
|
||||
// ADR 0605 §9: a build that names its artifacts in the seat's form is the same artifact: removed and held under the
|
||||
// seat's name and digest, and said gone in its own form.
|
||||
{name: "the seat's form", most: MostPerSweep, letGo: all, seatForm: true},
|
||||
// A seat that refuses a removal stops the sweep; the finding sweep-refused is written, and cleared by a clean run.
|
||||
{name: "a refused removal", most: MostPerSweep, letGo: []string{archiveRef("letta", 2)}, left: 2, refuse: "remove " + imageRef("letta", 2)},
|
||||
{name: "a confirmed collect", how: "collect", letGo: all},
|
||||
{name: "a sweep", most: MostPerSweep, letGo: all},
|
||||
{name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
|
||||
{name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
|
||||
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
|
||||
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
|
||||
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3, held: 8},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
h, b, door, acts := aMesh(t)
|
||||
door.indexes[imageRef("letta", 2)] = c.index
|
||||
if c.held == 0 {
|
||||
c.held = 7
|
||||
}
|
||||
if c.seatForm {
|
||||
for _, n := range []int{2, 3} {
|
||||
key, _ := builds.BuilderBuildKey("letta", fp(n))
|
||||
raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}})
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
}
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = map[string]error{c.refuse: errors.New("the store is read only")}
|
||||
}
|
||||
if c.unlog > 0 {
|
||||
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
|
||||
}
|
||||
@@ -284,7 +344,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != c.held || a.Missing != 0 {
|
||||
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
|
||||
}
|
||||
}
|
||||
@@ -298,8 +358,34 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
held++
|
||||
}
|
||||
}
|
||||
if held != 6 {
|
||||
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
|
||||
if held != c.held {
|
||||
t.Errorf("the seats were asked %v: the %d kept archives and copies are held first", door.asked, c.held)
|
||||
}
|
||||
// Each removal asks the seat that holds it, by its name and digest, for retention, after a release.
|
||||
for _, call := range door.calls {
|
||||
if strings.Contains(call, ".remove ") && !strings.HasSuffix(call, " "+Retention) {
|
||||
t.Errorf("a removal asked %q, not for retention", call)
|
||||
}
|
||||
}
|
||||
if c.letGo[0] == archiveRef("letta", 2) && !slices.Contains(door.calls, "seat:oci-artifact-registry.remove letta/files@"+digestOf(archiveRef("letta", 2))+" "+Retention) {
|
||||
t.Errorf("the seats were called %v", door.calls)
|
||||
}
|
||||
_, finding, _ := b.Get(FindingsState, sweepFindingKey)
|
||||
if finding != (c.refuse != "") {
|
||||
t.Errorf("the sweep's finding is written: %v, want %v", finding, c.refuse != "")
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = nil
|
||||
if _, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "again"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still, _ := b.Get(FindingsState, sweepFindingKey); still {
|
||||
t.Error("a clean sweep left the sweep's finding")
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.seatForm && !strings.Contains(b.published[0], `"container-image-registry/letta/image@`+digestOf(imageRef("letta", 2))+`"`) {
|
||||
t.Errorf("artifacts-gone does not name the build's own form: %v", b.published)
|
||||
}
|
||||
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
|
||||
!strings.Contains((*acts)[0], c.letGo[0]) {
|
||||
@@ -324,9 +410,9 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
if err := h.Observe(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing)
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "letta.ace", ImageMissing)
|
||||
raw, found, _ := b.Get(FindingsState, key)
|
||||
f, err := health.DecodeFinding(raw)
|
||||
f, err := meshconditions.DecodeFindingsByKeyValue(raw)
|
||||
if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) {
|
||||
t.Fatalf("the finding is %+v (%v, %v)", f, found, err)
|
||||
}
|
||||
@@ -346,7 +432,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
t.Fatalf("the pinnedImage image missing is %s", raw)
|
||||
}
|
||||
|
||||
have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"})
|
||||
have, _ := json.Marshal(nodeengine.HaveByNodeModuleValue{V: 1, Fp: fp(7), Phase: "running"})
|
||||
_ = b.Put(HavesRead, "ace.letta", json.RawMessage(have))
|
||||
images, err := h.ImagesOf("ace", "10.0.0.5:5000")
|
||||
if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) ||
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
// **The store, read through its seats** (novox/hq ADR 0632 §4): the read tools and the sweep read what the store holds
|
||||
// from the seats' `list`, container-image-registry's images and indexes and oci-artifact-registry's bundles and archives,
|
||||
// never from the store's files or its read side. What a manifest marks beyond itself is the seat's size of it, so a
|
||||
// blob two manifests share is counted in each, and what the store's nightly collector frees is not known before it runs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// Repo is one repository as the seats' list answers it.
|
||||
type Repo struct {
|
||||
Name string
|
||||
// Revisions are the digests of every manifest it holds, tagged or not.
|
||||
Revisions map[string]bool
|
||||
// Tags are what each tag names now.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Listing is the store as the seats' list answers it: every repository, and a size for each manifest and each
|
||||
// bundle's or archive's blob, keyed by its digest. A manifest's size is its own content and every blob it names, as the
|
||||
// seat answers it, so a layer two manifests share is counted in each.
|
||||
type Listing struct {
|
||||
Blobs map[string]int64
|
||||
Repos map[string]*Repo
|
||||
}
|
||||
|
||||
func (l *Listing) repo(name string) *Repo {
|
||||
r := l.Repos[name]
|
||||
if r == nil {
|
||||
r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
|
||||
l.Repos[name] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RepoNames are the repositories in order.
|
||||
func (l *Listing) RepoNames() []string {
|
||||
out := make([]string, 0, len(l.Repos))
|
||||
for n := range l.Repos {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Manifest is what one manifest names.
|
||||
type Manifest struct {
|
||||
MediaType string
|
||||
ConfigMedia string
|
||||
// Refs are the blobs it names: its configuration and its layers.
|
||||
Refs []string
|
||||
// Layers are its layers alone, in order.
|
||||
Layers []string
|
||||
// Children are the manifests an index names.
|
||||
Children []string
|
||||
}
|
||||
|
||||
// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
|
||||
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
|
||||
// document the same way, under a tag.
|
||||
func (m *Manifest) Holder() bool {
|
||||
return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
|
||||
}
|
||||
|
||||
// Key names one manifest in one repository.
|
||||
func Key(repo, digest string) string { return repo + "@" + digest }
|
||||
|
||||
// Store is the artifact store as its two seats' `list` answers it: Ask is the runtime's stdio.Ask in main.
|
||||
type Store struct {
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// ListOf asks seat's list, of one repository or, with name empty, of every one.
|
||||
func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) {
|
||||
if s == nil || s.Ask == nil {
|
||||
return nil, fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
}
|
||||
var a reg.ListAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
}
|
||||
return a.Repositories, nil
|
||||
}
|
||||
|
||||
// View reads the store now, through both seats.
|
||||
func (s *Store) View(context.Context) (*View, error) {
|
||||
images, err := s.ListOf(reg.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts, err := s.ListOf(ociseat.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ViewOf(images, artifacts), nil
|
||||
}
|
||||
|
||||
// ViewOf is the store as the two seats list it: each manifest a repository's revision, with its tags; an index naming its
|
||||
// platform manifests; a bundle's or archive's manifest keeping its one blob, so it reads as the holder of that archive.
|
||||
func ViewOf(seats ...[]reg.Repository) *View {
|
||||
l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
m := map[string]*Manifest{}
|
||||
held := map[string]bool{}
|
||||
for _, repos := range seats {
|
||||
for _, r := range repos {
|
||||
repo := l.repo(r.Name)
|
||||
for _, e := range r.Manifests {
|
||||
repo.Revisions[e.Digest] = true
|
||||
for _, t := range e.Tags {
|
||||
repo.Tags[t] = e.Digest
|
||||
}
|
||||
doc := &Manifest{Children: append([]string(nil), e.Platforms...)}
|
||||
l.Blobs[e.Digest] = e.Size
|
||||
if e.Artifact != "" {
|
||||
// The holder's size is the archive's with it: the blob itself counts nothing more.
|
||||
doc.ConfigMedia, doc.Layers, doc.Refs = emptyConfig, []string{e.Artifact}, []string{e.Artifact}
|
||||
if _, sized := l.Blobs[e.Artifact]; !sized {
|
||||
l.Blobs[e.Artifact] = 0
|
||||
}
|
||||
}
|
||||
m[Key(r.Name, e.Digest)] = doc
|
||||
if e.Held {
|
||||
held[Key(r.Name, e.Digest)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &View{L: l, M: m, Unread: map[string]string{}, Held: held}
|
||||
}
|
||||
|
||||
// emptyConfig is the configuration media type of a manifest keeping one blob.
|
||||
const emptyConfig = "application/vnd.oci.empty.v1+json"
|
||||
|
||||
// listed is the one entry of a reference's seat list naming it: by its manifest's digest, or a bundle's or an archive's by
|
||||
// the blob its reference names. Gone when the seat lists none.
|
||||
func (s *Store) listed(reference string) (reg.Listed, error) {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return reg.Listed{}, fmt.Errorf("%w: %v", ErrNotOurs, err)
|
||||
}
|
||||
repos, err := s.ListOf(seat, name)
|
||||
if err != nil {
|
||||
return reg.Listed{}, err
|
||||
}
|
||||
for _, r := range repos {
|
||||
for _, e := range r.Manifests {
|
||||
if (seat == ociseat.Seat && e.Artifact == digest) || (seat != ociseat.Seat && e.Digest == digest) {
|
||||
return e, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return reg.Listed{}, Gone
|
||||
}
|
||||
@@ -1,38 +1,46 @@
|
||||
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
|
||||
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
|
||||
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
|
||||
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
// size, and what the mesh keeps of each manifest — read through the registry seats' list (novox/hq ADR 0632 §4), and it
|
||||
// holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
//
|
||||
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
|
||||
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
|
||||
// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says
|
||||
// on stderr.
|
||||
// **A module beside the store, not the store's own.** It requires no provision: what it reads, holds and removes it asks
|
||||
// the registry seats (listed.go, sweepseats.go), never the store's files or its read side. stdout is the protocol; what this bundle says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
s := &Store{
|
||||
Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
|
||||
Run: ExecRunner,
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
|
||||
Now: time.Now, Logf: logf}
|
||||
h.Acts = handacts.Recorder{Ask: stdio.Ask}
|
||||
go follow(h)
|
||||
s := &Store{Ask: stdio.Ask}
|
||||
v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
|
||||
v.Follow(assignerModule, "wants", WantsTool)
|
||||
v.Follow(indexerModule, "manifests", ManifestsTool)
|
||||
h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE"), s), Listed: s.View,
|
||||
Seats: stdio.Ask, Now: time.Now, Logf: logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
v.Start(context.Background())
|
||||
go follow(h, v)
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
|
||||
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
|
||||
if err := stdio.Serve("", append(append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...), v.Tools("artifact_store")...)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// The owners' read tools of the states the holder follows, which the manifest invokes (ADR 0615).
|
||||
const (
|
||||
WantsTool = assignerModule + ".assigner_read_wants"
|
||||
ManifestsTool = indexerModule + ".indexer_read_manifests"
|
||||
)
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// readScript prints each manifest named on its command line, each after a line naming its path. A
|
||||
// manifest is JSON, which holds no raw line break inside a string, so no line of one starts with the
|
||||
// marker. One the store does not have is said, not skipped.
|
||||
const readScript = `cd ` + storageRoot + `
|
||||
for p in "$@"; do
|
||||
echo "#manifest $p"
|
||||
if [ -f "$p" ]; then cat "$p"; echo; else echo "#absent"; fi
|
||||
done
|
||||
echo '#end'`
|
||||
|
||||
// readBatch is how many manifests one exec reads: well under any command line's limit.
|
||||
const readBatch = 400
|
||||
|
||||
// Store reaches the artifact store's own files, through its own container, and only reads them.
|
||||
type Store struct {
|
||||
Container string
|
||||
Run Runner
|
||||
UID int
|
||||
|
||||
mu sync.Mutex
|
||||
// cache holds every manifest read: a manifest is named by its content's digest, so it never changes.
|
||||
cache map[string]*Manifest
|
||||
}
|
||||
|
||||
// List reads the store's files.
|
||||
func (s *Store) List(ctx context.Context) (*Listing, error) {
|
||||
if s.Container == "" {
|
||||
return nil, fmt.Errorf("this bundle was not told the artifact store's container (MESH_ARTIFACT_STORE_CONTAINER)")
|
||||
}
|
||||
out, err := docker(ctx, s.Run, s.UID, "exec", s.Container, "sh", "-c", listScript)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseListing(out)
|
||||
}
|
||||
|
||||
// Key names one manifest in one repository.
|
||||
func Key(repo, digest string) string { return repo + "@" + digest }
|
||||
|
||||
// blobPath is where the store keeps a blob, under storageRoot.
|
||||
func blobPath(digest string) string {
|
||||
algorithm, hex, _ := strings.Cut(digest, ":")
|
||||
if algorithm == "" || len(hex) < 2 || strings.ContainsAny(digest, "/ \n") {
|
||||
return ""
|
||||
}
|
||||
return "blobs/" + algorithm + "/" + hex[:2] + "/" + hex + "/data"
|
||||
}
|
||||
|
||||
// Manifests reads every manifest the listing names, from the cache or the store's files. Answers each
|
||||
// one read, and each one not read with why.
|
||||
func (s *Store) Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string) {
|
||||
need := map[string]bool{}
|
||||
s.mu.Lock()
|
||||
if s.cache == nil {
|
||||
s.cache = map[string]*Manifest{}
|
||||
}
|
||||
for _, name := range l.RepoNames() {
|
||||
for d := range l.Repos[name].Revisions {
|
||||
if _, ok := s.cache[d]; !ok {
|
||||
need[d] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
digests := make([]string, 0, len(need))
|
||||
for d := range need {
|
||||
digests = append(digests, d)
|
||||
}
|
||||
sort.Strings(digests)
|
||||
failed := map[string]string{}
|
||||
for start := 0; start < len(digests); start += readBatch {
|
||||
batch := digests[start:min(start+readBatch, len(digests))]
|
||||
read, err := s.read(ctx, batch)
|
||||
s.mu.Lock()
|
||||
for _, d := range batch {
|
||||
switch m := read[d]; {
|
||||
case err != nil:
|
||||
failed[d] = err.Error()
|
||||
case m == nil:
|
||||
failed[d] = "the store holds no manifest content for it"
|
||||
default:
|
||||
s.cache[d] = m
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
got := map[string]*Manifest{}
|
||||
unread := map[string]string{}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, name := range l.RepoNames() {
|
||||
for d := range l.Repos[name].Revisions {
|
||||
if m, ok := s.cache[d]; ok {
|
||||
got[Key(name, d)] = m
|
||||
} else if why, ok := failed[d]; ok {
|
||||
unread[Key(name, d)] = why
|
||||
} else {
|
||||
unread[Key(name, d)] = "not read"
|
||||
}
|
||||
}
|
||||
}
|
||||
return got, unread
|
||||
}
|
||||
|
||||
// read reads one batch of manifests. A blob that is not a manifest, or not there, is answered as nil;
|
||||
// an exec that fails fails the batch.
|
||||
func (s *Store) read(ctx context.Context, digests []string) (map[string]*Manifest, error) {
|
||||
args := []string{"exec", s.Container, "sh", "-c", readScript, "sh"}
|
||||
byPath := map[string]string{}
|
||||
for _, d := range digests {
|
||||
if p := blobPath(d); p != "" {
|
||||
byPath[p] = d
|
||||
args = append(args, p)
|
||||
}
|
||||
}
|
||||
out, err := docker(ctx, s.Run, s.UID, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseManifests(out, byPath)
|
||||
}
|
||||
|
||||
// parseManifests reads readScript's output.
|
||||
func parseManifests(out string, byPath map[string]string) (map[string]*Manifest, error) {
|
||||
got := map[string]*Manifest{}
|
||||
current := ""
|
||||
var body strings.Builder
|
||||
flush := func() {
|
||||
if d := byPath[current]; d != "" {
|
||||
m, err := ParseManifest([]byte(body.String()))
|
||||
if err != nil {
|
||||
m = nil
|
||||
}
|
||||
got[d] = m
|
||||
}
|
||||
current = ""
|
||||
body.Reset()
|
||||
}
|
||||
ended := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
switch {
|
||||
case strings.HasPrefix(line, "#manifest "):
|
||||
flush()
|
||||
current = strings.TrimPrefix(line, "#manifest ")
|
||||
case line == "#absent":
|
||||
if d := byPath[current]; d != "" {
|
||||
got[d] = nil
|
||||
}
|
||||
current = ""
|
||||
body.Reset()
|
||||
case line == "#end":
|
||||
flush()
|
||||
ended = true
|
||||
default:
|
||||
body.WriteString(line)
|
||||
body.WriteString("\n")
|
||||
}
|
||||
}
|
||||
if !ended {
|
||||
return nil, fmt.Errorf("reading the store's manifests was cut short (it never reached its end)")
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestManifestsAreReadFromTheStoresFilesAndAMissingOneIsSaid(t *testing.T) {
|
||||
got, err := parseManifests("#manifest blobs/sha256/aa/aa1/data\n"+image(d("cfg"), d("l"))+"\n\n#manifest blobs/sha256/bb/bb1/data\n#absent\n#manifest blobs/sha256/cc/cc1/data\nnot json\n#end\n",
|
||||
map[string]string{"blobs/sha256/aa/aa1/data": "sha256:aa1", "blobs/sha256/bb/bb1/data": "sha256:bb1", "blobs/sha256/cc/cc1/data": "sha256:cc1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := got["sha256:aa1"]; m == nil || len(m.Layers) != 1 || m.Layers[0] != d("l") {
|
||||
t.Errorf("aa1 %+v", m)
|
||||
}
|
||||
if got["sha256:bb1"] != nil || got["sha256:cc1"] != nil {
|
||||
t.Error("an absent or unreadable manifest was read as one")
|
||||
}
|
||||
if _, err := parseManifests("#manifest x\n{}\n", map[string]string{"x": "sha256:x"}); err == nil {
|
||||
t.Error("a cut read was accepted")
|
||||
}
|
||||
|
||||
f := world()
|
||||
delete(f.manifests, d("m-old"))
|
||||
v := view(t, f)
|
||||
if v.Unread[Key("app/server", d("m-old"))] == "" {
|
||||
t.Error("a manifest whose content is gone was not said as unread")
|
||||
}
|
||||
if out := Usage(v, 3); out["unread"] == nil {
|
||||
t.Error("the answer did not say a manifest was unread")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADigestNeverBecomesAPathOutsideTheBlobs(t *testing.T) {
|
||||
for _, bad := range []string{"sha256:", "nocolon", "sha256:a/../../etc", "sha256:ab cd"} {
|
||||
if p := blobPath(bad); p != "" {
|
||||
t.Errorf("%q became %q", bad, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
|
||||
type Ran struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Status int
|
||||
// Err is "ENOENT" when the program is not installed, or says it was ended for taking too long.
|
||||
Err string
|
||||
}
|
||||
|
||||
// Runner runs one command, so every tool can be tested without a daemon.
|
||||
type Runner func(ctx context.Context, name string, args ...string) Ran
|
||||
|
||||
// ListTimeout is how long listing the store's files may take: below the runtime's thirty-second call
|
||||
// limit, so a store that hangs is answered as such rather than as a call the runtime gave up on.
|
||||
const ListTimeout = 15 * time.Second
|
||||
|
||||
// ExecRunner runs a command on this machine, bounded by ListTimeout.
|
||||
func ExecRunner(ctx context.Context, name string, args ...string) Ran {
|
||||
ctx, cancel := context.WithTimeout(ctx, ListTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||
err := cmd.Run()
|
||||
r := Ran{Stdout: out.String(), Stderr: errb.String()}
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
r.Status, r.Err = 124, fmt.Sprintf("no answer within %d s", int(ListTimeout/time.Second))
|
||||
case errors.Is(err, exec.ErrNotFound):
|
||||
r.Status, r.Err = 127, "ENOENT"
|
||||
case errors.As(err, &exitErr):
|
||||
r.Status = exitErr.ExitCode()
|
||||
case err != nil:
|
||||
r.Status, r.Err = 1, err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
var socketRefused = regexp.MustCompile(`(?i)permission denied.*docker.*sock|docker\.sock.*permission denied`)
|
||||
|
||||
// docker runs one docker command as this account and answers its stdout. A socket that refuses the
|
||||
// account is asked again through `sudo -n`, as the container runtime's own tools do; never as root
|
||||
// otherwise, and never with a prompt.
|
||||
func docker(ctx context.Context, run Runner, uid int, args ...string) (string, error) {
|
||||
r := run(ctx, "docker", args...)
|
||||
program := "docker"
|
||||
if r.Status != 0 && r.Err == "" && uid != 0 && socketRefused.MatchString(r.Stderr) {
|
||||
program = "sudo"
|
||||
r = run(ctx, "sudo", append([]string{"-n", "docker"}, args...)...)
|
||||
}
|
||||
if r.Status == 0 && r.Err == "" {
|
||||
return r.Stdout, nil
|
||||
}
|
||||
said := strings.TrimSpace(r.Stderr + "\n" + r.Stdout)
|
||||
switch {
|
||||
case r.Err == "ENOENT" && program == "sudo":
|
||||
return "", errors.New("the runtime's socket refused this account, and sudo is not installed here to escalate with")
|
||||
case r.Err == "ENOENT":
|
||||
return "", errors.New("docker is not installed on this machine, so the store's files cannot be listed")
|
||||
case r.Err != "":
|
||||
return "", fmt.Errorf("listing the store's files did not answer: %s", r.Err)
|
||||
case program == "sudo" && strings.HasPrefix(said, "sudo:"):
|
||||
return "", fmt.Errorf("the runtime's socket refused this account and it may not escalate without a prompt: %s", firstLine(said))
|
||||
case strings.Contains(said, "No such container"):
|
||||
return "", fmt.Errorf("the store's container is not on this machine: %s", firstLine(said))
|
||||
case strings.Contains(said, "is not running"):
|
||||
return "", fmt.Errorf("the store's container is not running: %s", firstLine(said))
|
||||
}
|
||||
if l := firstLine(said); l != "" {
|
||||
return "", fmt.Errorf("listing the store's files failed (%d): %s", r.Status, l)
|
||||
}
|
||||
return "", fmt.Errorf("listing the store's files failed with status %d", r.Status)
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named
|
||||
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; sweep.Seat), each a tool named
|
||||
// `artifact-store.<verb>` as the runtime serves a seat's verb, answering as the controller's verb answered: {output,
|
||||
// ok, answer}.
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf).
|
||||
@@ -21,7 +21,7 @@ type verbAnswer struct {
|
||||
|
||||
// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors.
|
||||
func SeatTools(h *Holder, s *Store) []stdio.Tool {
|
||||
seat := catalogue.ArtifactSweepSeat + "."
|
||||
seat := sweep.Seat + "."
|
||||
return []stdio.Tool{
|
||||
{Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " +
|
||||
"or with collected those it no longer holds.",
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the artifact store holds, read from its own files, and nothing written to them.
|
||||
//
|
||||
// **Why its files.** The store's door lists repositories and the tags in each, but not a manifest no tag
|
||||
// names — and since the mesh pins every machine by digest, that is almost every manifest the store
|
||||
// holds. The files list them all, with every blob's size, and hold every manifest's content. They are
|
||||
// read through the store's own container, which is the only process that has them, and only read
|
||||
// (novox/hq ADR 0251 §1). So this module runs on the machine that holds the store; anywhere else its
|
||||
// tools say the container is not there.
|
||||
|
||||
// storageRoot is where the registry keeps its files, inside its container.
|
||||
const storageRoot = "/var/lib/registry/docker/registry/v2"
|
||||
|
||||
// listScript lists, in three sections, every blob with its size, every manifest each repository holds,
|
||||
// and what each tag names now. Busybox's find and stat (the store's image is Alpine): no -printf. An
|
||||
// upload in progress is not listed, and nor are a tag's former targets.
|
||||
const listScript = `set -e
|
||||
cd ` + storageRoot + `
|
||||
echo '#blobs'
|
||||
if [ -d blobs ]; then find blobs -type f -name data | xargs -r stat -c '%s %n'; fi
|
||||
echo '#revisions'
|
||||
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/revisions/sha256/*/link'; fi
|
||||
echo '#tags'
|
||||
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/tags/*/current/link' -exec grep -H . {} + || true; fi
|
||||
echo '#end'`
|
||||
|
||||
// Repo is one repository as the store's files list it.
|
||||
type Repo struct {
|
||||
Name string
|
||||
// Revisions are the digests of every manifest it holds, tagged or not.
|
||||
Revisions map[string]bool
|
||||
// Tags are what each tag names now.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Listing is the store's files: every blob with its size, and every repository.
|
||||
type Listing struct {
|
||||
Blobs map[string]int64
|
||||
Repos map[string]*Repo
|
||||
}
|
||||
|
||||
func (l *Listing) repo(name string) *Repo {
|
||||
r := l.Repos[name]
|
||||
if r == nil {
|
||||
r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
|
||||
l.Repos[name] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RepoNames are the repositories in order.
|
||||
func (l *Listing) RepoNames() []string {
|
||||
out := make([]string, 0, len(l.Repos))
|
||||
for n := range l.Repos {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseListing reads listScript's output. A section that never ended means the listing was cut short,
|
||||
// which is an error: a partial listing would make held things look absent.
|
||||
func ParseListing(out string) (*Listing, error) {
|
||||
l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
section := ""
|
||||
ended := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") {
|
||||
section = line
|
||||
ended = line == "#end"
|
||||
continue
|
||||
}
|
||||
switch section {
|
||||
case "#blobs":
|
||||
size, path, ok := strings.Cut(line, " ")
|
||||
n, err := strconv.ParseInt(size, 10, 64)
|
||||
if !ok || err != nil {
|
||||
return nil, fmt.Errorf("the store's listing has a blob line it cannot read: %q", line)
|
||||
}
|
||||
// blobs/sha256/ab/<hex>/data
|
||||
parts := strings.Split(strings.TrimPrefix(path, "./"), "/")
|
||||
if len(parts) != 5 || parts[0] != "blobs" || parts[4] != "data" {
|
||||
return nil, fmt.Errorf("the store's listing has a blob at an unexpected place: %q", path)
|
||||
}
|
||||
l.Blobs[parts[1]+":"+parts[3]] = n
|
||||
case "#revisions":
|
||||
// repositories/<repo>/_manifests/revisions/sha256/<hex>/link
|
||||
name, rest, ok := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(line, "./"), "repositories/"), "/_manifests/revisions/")
|
||||
parts := strings.Split(rest, "/")
|
||||
if !ok || len(parts) != 3 || parts[2] != "link" {
|
||||
return nil, fmt.Errorf("the store's listing has a manifest at an unexpected place: %q", line)
|
||||
}
|
||||
l.repo(name).Revisions[parts[0]+":"+parts[1]] = true
|
||||
case "#tags":
|
||||
// repositories/<repo>/_manifests/tags/<tag>/current/link:sha256:<hex>
|
||||
path, digest, ok := strings.Cut(line, ":sha256:")
|
||||
name, rest, ok2 := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(path, "./"), "repositories/"), "/_manifests/tags/")
|
||||
tag, _, ok3 := strings.Cut(rest, "/current/link")
|
||||
if !ok || !ok2 || !ok3 || tag == "" {
|
||||
return nil, fmt.Errorf("the store's listing has a tag line it cannot read: %q", line)
|
||||
}
|
||||
l.repo(name).Tags[tag] = "sha256:" + digest
|
||||
default:
|
||||
return nil, fmt.Errorf("the store's listing has a line outside any section: %q", line)
|
||||
}
|
||||
}
|
||||
if !ended {
|
||||
return nil, fmt.Errorf("the store's listing was cut short (it never reached its end)")
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Manifest is what one manifest names.
|
||||
type Manifest struct {
|
||||
MediaType string
|
||||
ConfigMedia string
|
||||
// Refs are the blobs it names: its configuration and its layers.
|
||||
Refs []string
|
||||
// Layers are its layers alone, in order.
|
||||
Layers []string
|
||||
// Children are the manifests an index names.
|
||||
Children []string
|
||||
}
|
||||
|
||||
// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
|
||||
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
|
||||
// document the same way, under a tag.
|
||||
func (m *Manifest) Holder() bool {
|
||||
return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
|
||||
}
|
||||
|
||||
// ParseManifest reads a manifest's content, of any of the shapes a registry holds.
|
||||
func ParseManifest(raw []byte) (*Manifest, error) {
|
||||
type desc struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
var v struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Config *desc `json:"config"`
|
||||
Layers []desc `json:"layers"`
|
||||
Manifests []desc `json:"manifests"`
|
||||
FSLayers []struct {
|
||||
BlobSum string `json:"blobSum"`
|
||||
} `json:"fsLayers"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, fmt.Errorf("not a manifest: %v", err)
|
||||
}
|
||||
m := &Manifest{MediaType: v.MediaType}
|
||||
if v.Config != nil && v.Config.Digest != "" {
|
||||
m.ConfigMedia = v.Config.MediaType
|
||||
m.Refs = append(m.Refs, v.Config.Digest)
|
||||
}
|
||||
for _, d := range v.Layers {
|
||||
m.Refs = append(m.Refs, d.Digest)
|
||||
m.Layers = append(m.Layers, d.Digest)
|
||||
}
|
||||
for _, d := range v.FSLayers {
|
||||
m.Refs = append(m.Refs, d.BlobSum)
|
||||
m.Layers = append(m.Layers, d.BlobSum)
|
||||
}
|
||||
for _, d := range v.Manifests {
|
||||
m.Children = append(m.Children, d.Digest)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
@@ -1,13 +1,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -22,66 +27,72 @@ func d(name string) string {
|
||||
|
||||
func hexOf(digest string) string { return strings.TrimPrefix(digest, "sha256:") }
|
||||
|
||||
// fakeStore is a store's files and its manifests: what listScript and readScript would print.
|
||||
// fakeStore is a store's blobs and manifests, as the two registry seats' list answers them (ADR 0632 §4).
|
||||
type fakeStore struct {
|
||||
blobs map[string]int64
|
||||
revisions map[string][]string // repo -> digests
|
||||
tags map[string]map[string]string
|
||||
manifests map[string]string // digest -> content
|
||||
held map[string]bool // Key(repo, digest) -> held
|
||||
}
|
||||
|
||||
func (f *fakeStore) listing() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("#blobs\n")
|
||||
for dg, n := range f.blobs {
|
||||
fmt.Fprintf(&b, "%d blobs/sha256/%s/%s/data\n", n, hexOf(dg)[:2], hexOf(dg))
|
||||
}
|
||||
b.WriteString("#revisions\n")
|
||||
for repo, ds := range f.revisions {
|
||||
for _, dg := range ds {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/revisions/sha256/%s/link\n", repo, hexOf(dg))
|
||||
// ask is the seats as the runtime asks them: list on each, every repository or one; anything else fails the test, as
|
||||
// these tools only read.
|
||||
func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error) {
|
||||
return func(key string, body any) (json.RawMessage, error) {
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
if verb != "list" || (seat != "container-image-registry" && seat != "oci-artifact-registry") {
|
||||
t.Fatalf("asked %s", key)
|
||||
}
|
||||
}
|
||||
b.WriteString("#tags\n")
|
||||
for repo, ts := range f.tags {
|
||||
for t, dg := range ts {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/tags/%s/current/link:%s\n", repo, t, dg)
|
||||
raw, _ := json.Marshal(body)
|
||||
var in reg.ListInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
repos := []reg.Repository{}
|
||||
names := make([]string, 0, len(f.revisions))
|
||||
for n := range f.revisions {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// run is the store's container as docker exec reaches it: the listing, or the manifests asked for, as
|
||||
// the two scripts print them. Anything else is a test failure: these tools only read.
|
||||
func (f *fakeStore) run(t *testing.T) Runner {
|
||||
return func(_ context.Context, name string, args ...string) Ran {
|
||||
if name != "docker" || len(args) < 5 || args[0] != "exec" || args[1] != "mesh-registry" || args[2] != "sh" || args[3] != "-c" {
|
||||
t.Fatalf("ran %s %v", name, args)
|
||||
}
|
||||
switch args[4] {
|
||||
case listScript:
|
||||
return Ran{Stdout: f.listing()}
|
||||
case readScript:
|
||||
var b strings.Builder
|
||||
for _, p := range args[6:] {
|
||||
fmt.Fprintf(&b, "#manifest %s\n", p)
|
||||
found := false
|
||||
for dg, body := range f.manifests {
|
||||
if blobPath(dg) == p {
|
||||
b.WriteString(body + "\n")
|
||||
found = true
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if in.Name != "" && in.Name != name {
|
||||
continue
|
||||
}
|
||||
var ms []reg.Listed
|
||||
for _, dg := range f.revisions[name] {
|
||||
var doc struct {
|
||||
Config *struct{ MediaType, Digest string }
|
||||
Layers []struct{ Digest string }
|
||||
Manifests []struct{ Digest string }
|
||||
}
|
||||
_ = json.Unmarshal([]byte(f.manifests[dg]), &doc)
|
||||
e := reg.Listed{Digest: dg, Size: f.blobs[dg], Held: f.held[Key(name, dg)]}
|
||||
if doc.Config != nil {
|
||||
e.Size += f.blobs[doc.Config.Digest]
|
||||
}
|
||||
for _, l := range doc.Layers {
|
||||
e.Size += f.blobs[l.Digest]
|
||||
}
|
||||
for _, c := range doc.Manifests {
|
||||
e.Platforms = append(e.Platforms, c.Digest)
|
||||
}
|
||||
e.Index = len(e.Platforms) > 0
|
||||
if doc.Config != nil && doc.Config.MediaType == "application/vnd.oci.empty.v1+json" && len(doc.Layers) == 1 {
|
||||
e.Artifact = doc.Layers[0].Digest
|
||||
}
|
||||
for tag, at := range f.tags[name] {
|
||||
if at == dg {
|
||||
e.Tags = append(e.Tags, tag)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
b.WriteString("#absent\n")
|
||||
if (e.Artifact != "") == (seat == "oci-artifact-registry") {
|
||||
ms = append(ms, e)
|
||||
}
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return Ran{Stdout: b.String()}
|
||||
if len(ms) > 0 {
|
||||
repos = append(repos, reg.Repository{Name: name, Manifests: ms})
|
||||
}
|
||||
}
|
||||
t.Fatalf("ran a script these tools do not have: %q", args[4])
|
||||
return Ran{}
|
||||
return json.Marshal(reg.ListAnswer{Repositories: repos})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,7 +164,7 @@ func records() *Records {
|
||||
|
||||
func view(t *testing.T, f *fakeStore) *View {
|
||||
t.Helper()
|
||||
s := &Store{Container: "mesh-registry", UID: 1000, Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
v, err := s.View(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -168,50 +179,34 @@ func withFill(r *Records) *Records {
|
||||
return r
|
||||
}
|
||||
|
||||
func TestTheListingIsRead(t *testing.T) {
|
||||
l, err := ParseListing("#blobs\n12 blobs/sha256/ab/abcd/data\n#revisions\nrepositories/a/b/c/_manifests/revisions/sha256/ff/link\n#tags\nrepositories/a/b/c/_manifests/tags/v1/current/link:sha256:ff\n#end\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// The store is read through its two seats' list (ADR 0632 §4): every manifest of every repository, tagged or not, an
|
||||
// index with the manifests it names, a bundle's or archive's manifest as the holder of its blob; each manifest's size is
|
||||
// the seat's, so nothing is unmarked before a manifest goes, and a manifest gone frees its size.
|
||||
func TestTheStoreIsReadThroughItsSeats(t *testing.T) {
|
||||
f := world()
|
||||
f.held = map[string]bool{Key("app/tools", d("h-kept")): true}
|
||||
v := view(t, f)
|
||||
server := v.L.Repos["app/server"]
|
||||
if server == nil || len(server.Revisions) != 4 || server.Tags["latest"] != d("m-stranger") {
|
||||
t.Fatalf("app/server reads %+v", server)
|
||||
}
|
||||
if l.Blobs["sha256:abcd"] != 12 {
|
||||
t.Errorf("blobs %v", l.Blobs)
|
||||
if m := v.M[Key("app/server", d("idx"))]; m == nil || !reflect.DeepEqual(m.Children, []string{d("m-kept")}) {
|
||||
t.Errorf("the index names %+v", m)
|
||||
}
|
||||
r := l.Repos["a/b/c"]
|
||||
if r == nil || !r.Revisions["sha256:ff"] || r.Tags["v1"] != "sha256:ff" {
|
||||
t.Errorf("repository with slashes misread: %+v", r)
|
||||
if m := v.M[Key("app/tools", d("h-kept"))]; !m.Holder() || m.Layers[0] != d("a-kept") {
|
||||
t.Errorf("the holder of a-kept reads %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACutListingIsRefused(t *testing.T) {
|
||||
for _, out := range []string{"#blobs\n12 blobs/sha256/ab/abcd/data\n", "#blobs\nnot a line\n#end\n", "stray\n#end\n"} {
|
||||
if _, err := ParseListing(out); err == nil {
|
||||
t.Errorf("accepted %q", out)
|
||||
}
|
||||
if _, held := v.L.Blobs[d("a-kept")]; !held || !v.Held[Key("app/tools", d("h-kept"))] {
|
||||
t.Errorf("a-kept is not read as held by the store, or its hold is not read: %v", v.Held)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMarkIsTheCollectorsIncludingAnIndexAndSharedBlobs(t *testing.T) {
|
||||
v := view(t, world())
|
||||
marked := v.Marked(nil)
|
||||
for _, name := range []string{"cfg", "shared", "old-only", "stranger-layer", "a-kept", "a-old", "doc-body", "empty", "m-kept", "idx"} {
|
||||
if !marked[d(name)] {
|
||||
t.Errorf("%s not marked", name)
|
||||
}
|
||||
if want := int64(len(f.manifests[d("m-old")])) + 10 + 1000 + 500; v.L.Blobs[d("m-old")] != want {
|
||||
t.Errorf("m-old's size %d, want %d (its content and every blob it names)", v.L.Blobs[d("m-old")], want)
|
||||
}
|
||||
if marked[d("orphan")] {
|
||||
t.Error("a blob no manifest names is marked")
|
||||
if freed, n := v.Unmarked(nil); freed != 0 || n != 0 {
|
||||
t.Errorf("unmarked %d in %d before anything goes", freed, n)
|
||||
}
|
||||
freed, n := v.Unmarked(nil)
|
||||
if freed != 7000 || n != 1 {
|
||||
t.Errorf("the collector frees %d in %d, want 7000 in 1", freed, n)
|
||||
}
|
||||
// The index alone gone frees its own content and nothing m-kept still marks.
|
||||
_, n = v.Unmarked(map[string]bool{Key("app/server", d("idx")): true})
|
||||
if n != 2 {
|
||||
t.Errorf("without the index %d blobs unmarked, want 2 (orphan and the index itself)", n)
|
||||
}
|
||||
if sh := v.Shared(); !sh[d("empty")] || sh[d("cfg")] {
|
||||
t.Error("shared misjudged")
|
||||
if freed, n := v.Unmarked(map[string]bool{Key("app/server", d("idx")): true}); freed != v.L.Blobs[d("idx")] || n != 1 {
|
||||
t.Errorf("the index gone frees %d in %d, want its size", freed, n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -255,16 +250,15 @@ func TestAStateThisBundleDoesNotKnowIsKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
|
||||
func TestBytesFreedAreTheSizesOfWhatGoes(t *testing.T) {
|
||||
v := view(t, world())
|
||||
a := &CollectAnswer{DryRun: true, Eligible: 2, WouldLetGo: []string{
|
||||
"artifact-store://app/server@" + d("m-old"), "artifact-store://app/tools/blobs/" + d("a-old")}}
|
||||
out := Collected(v, a, true, nil)
|
||||
now, after := out["collector_frees_now_bytes"].(int64), out["collector_frees_after_bytes"].(int64)
|
||||
// m-old's own content, old-only and a-old and its holder go; shared and cfg stay with m-kept.
|
||||
want := int64(7000) + 500 + 4000 + v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]
|
||||
if now != 7000 || after != want {
|
||||
t.Errorf("now %d after %d, want 7000 and %d", now, after, want)
|
||||
// m-old and the holder of a-old go, each with the size its seat says.
|
||||
if want := v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]; now != 0 || after != want {
|
||||
t.Errorf("now %d after %d, want 0 and %d", now, after, want)
|
||||
}
|
||||
if out["manifests_gone"].(int) != 2 {
|
||||
t.Errorf("gone %v", out["manifests_gone"])
|
||||
@@ -275,8 +269,7 @@ func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
v := view(t, world())
|
||||
entries := Classify(v, withFill(records()), nil)
|
||||
sums := Summarise(v, entries, entries)
|
||||
el := sums[StateEligible]
|
||||
if el.OnlyTheirBytes != 500+v.L.Blobs[d("m-old")] {
|
||||
if el := sums[StateEligible]; el.OnlyTheirBytes != v.L.Blobs[d("m-old")] {
|
||||
t.Errorf("eligible only-theirs %d", el.OnlyTheirBytes)
|
||||
}
|
||||
}
|
||||
@@ -366,8 +359,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
if _, has := args["commit"]; has {
|
||||
return nil, fmt.Errorf("asked %s at %v: the seat's verb takes no commit", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
@@ -375,7 +368,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
raw, _ := json.Marshal(catalogue.CatalogOne{Module: catalogue.CatalogModule{
|
||||
CatalogEntry: catalogue.CatalogEntry{Module: name, Commit: "c-" + name}, Manifest: json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "collect":
|
||||
if args["confirm"] == "true" {
|
||||
@@ -392,7 +386,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
@@ -442,10 +436,12 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Claims []struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Requires []any `json:"requires"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
@@ -454,7 +450,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var served []string
|
||||
for _, tl := range Tools(&Store{}, recordsAsked{&asked{}}, &Catalogue{}) {
|
||||
for _, tl := range append(Tools(&Store{}, recordsAsked{&asked{}}, &Catalogue{}), (&views.Reader{}).Tools("artifact_store")...) {
|
||||
if !strings.HasPrefix(tl.Name, "artifact_store_") || tl.Description == "" || tl.Run == nil {
|
||||
t.Errorf("tool %q", tl.Name)
|
||||
}
|
||||
@@ -468,20 +464,41 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
}
|
||||
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
|
||||
// it reads no controller state, the lease neither (issue 1126).
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
|
||||
// CAT-AS1: what it holds and removes it asks the two registry seats the artifact store's holder serves.
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool,
|
||||
"seat:oci-artifact-registry.hold", "seat:oci-artifact-registry.release", "seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove",
|
||||
// ADR 0632 §4: what the store holds it reads through the two seats' list.
|
||||
"seat:container-image-registry.list", "seat:oci-artifact-registry.list"}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
// ADR 0615: the assigner's wants and the indexer's manifests are asked once and followed by their events, not read.
|
||||
if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests"}) ||
|
||||
slices.Contains(m.Reads, WantsRead) || slices.Contains(m.Reads, ManifestsRead) {
|
||||
t.Errorf("consumes %v, reads %v", m.Consumes, m.Reads)
|
||||
}
|
||||
// The operator, 2026-11-04 (issue 1966): it reaches the store through the registry seats' verbs, and requires no
|
||||
// provision artifact-store.
|
||||
if len(m.Requires) != 0 {
|
||||
t.Errorf("requires %v, want nothing: the store is reached through the registry seats", m.Requires)
|
||||
}
|
||||
// Its address file is filled from the image seat holder's listen (ADR 0632 §1), never by the retired provision.
|
||||
address, err := os.ReadFile("../../files/store-address")
|
||||
if err != nil || strings.TrimSpace(string(address)) != "${seat:container-image-registry:reach}" {
|
||||
t.Errorf("the store's address file reads %q (%v), want the image seat's reach", address, err)
|
||||
}
|
||||
var verbs []string
|
||||
for _, tl := range SeatTools(&Holder{}, &Store{}) {
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, sweep.Seat+"."))
|
||||
}
|
||||
seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat)
|
||||
catalogue.UseSeats(catalogue.SeatsFromDefinitions(seatlisttest.Definitions()))
|
||||
seat, _ := catalogue.SeatNamed(sweep.Seat)
|
||||
var contract []string
|
||||
for _, v := range seat.Serves {
|
||||
contract = append(contract, v.Name)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != sweep.Seat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
|
||||
!reflect.DeepEqual(verbs, contract) {
|
||||
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
|
||||
}
|
||||
@@ -491,25 +508,3 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) Ran {
|
||||
ran = append(ran, name)
|
||||
if name == "docker" {
|
||||
return Ran{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock"}
|
||||
}
|
||||
return Ran{Stdout: "#end\n"}
|
||||
}
|
||||
if _, err := docker(context.Background(), run, 1000, "exec", "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(ran, []string{"docker", "sudo"}) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
if _, err := docker(context.Background(), func(context.Context, string, ...string) Ran {
|
||||
return Ran{Status: 1, Stderr: "Error: No such container: mesh-registry"}
|
||||
}, 1000, "exec"); err == nil || !strings.Contains(err.Error(), "not on this machine") {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
// **The sweep asks the registry seats** (CAT-AS1; novox/hq ADR 0617 §2, design 51): the artifact store's write side is
|
||||
// its holder's, so the sweep no longer deletes or holds over the store's door. What it keeps from retention it asks the
|
||||
// seat's `hold` for (a kept bundle or archive on oci-artifact-registry, a kept copy of an upstream base on
|
||||
// container-image-registry), and what is eligible it asks the seat's `remove` for, with the sweep's reason; both verbs
|
||||
// are in the manifest's `invokes`, and only this module is on their caller lists. A reference is read in both forms
|
||||
// (ADR 0605 §9): the older `artifact-store://<path>` and the seat's `<seat>/<module>/<artifact>@<digest>`; the keep set
|
||||
// keys each by the older form, so the two meet as one artifact. A call a seat refuses stops the sweep and writes the
|
||||
// sweep's finding, sweep-refused; the next sweep with no refusal clears it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked.
|
||||
const SweepRefused = "sweep-refused"
|
||||
|
||||
// sweepFindingKey is the one key of the sweep's finding in the module's findings.
|
||||
var sweepFindingKey = meshconditionsclient.Key(meshconditionsclient.ScopeModule, Module, SweepRefused)
|
||||
|
||||
// canonical is a kept reference in the older form, `artifact-store://<path>`, whichever form it came in; another is as
|
||||
// it was.
|
||||
func canonical(reference string) string {
|
||||
path, kept := registryForms.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
return catalogue.ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// seatOf is the seat, name and digest a kept reference is held under: a bundle or an archive on oci-artifact-registry,
|
||||
// an image or a copy of an upstream base on container-image-registry.
|
||||
func seatOf(reference string) (seat, name, digest string, err error) {
|
||||
path, kept := registryForms.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", "", fmt.Errorf("%s is no reference into the mesh's registries", reference)
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "/blobs/"); ok {
|
||||
return ociseat.Seat, name, digest, nil
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "@"); ok {
|
||||
return reg.Seat, name, digest, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%s names no digest", reference)
|
||||
}
|
||||
|
||||
// isCopy is whether a kept reference is a copy of an upstream base.
|
||||
func isCopy(reference string) bool {
|
||||
path, _ := registryForms.InArtifactStore(reference)
|
||||
return strings.HasPrefix(path, "upstream/")
|
||||
}
|
||||
|
||||
// askSeat asks verb of the seat reference is held on, with body built from its name and digest.
|
||||
func (h *Holder) askSeat(reference, verb string, body func(name, digest string) any, answer any) error {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if h.Seats == nil {
|
||||
return fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := h.Seats("seat:"+seat+"."+verb, body(name, digest))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's %s of %s@%s: %w", seat, verb, name, digest, err)
|
||||
}
|
||||
if answer != nil && len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, answer); err != nil {
|
||||
return fmt.Errorf("%s's answer to %s: %w", seat, verb, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// holdOnSeat asks the seat to keep reference from retention, for why.
|
||||
func (h *Holder) holdOnSeat(reference, why string) (bool, error) {
|
||||
var a reg.HoldAnswer
|
||||
err := h.askSeat(reference, reg.VerbHold, func(name, digest string) any {
|
||||
return reg.HoldInput{Name: name, Digest: digest, Reason: why}
|
||||
}, &a)
|
||||
if err == nil && !a.Held {
|
||||
err = fmt.Errorf("%s: the seat answered it is not held", reference)
|
||||
}
|
||||
return a.Held, err
|
||||
}
|
||||
|
||||
// Retention is the reason of a removal by digest, the sweep's alone (design 51 *Who may call which verb*): the holder
|
||||
// removes an untagged thing only for it. The seat's definition names it.
|
||||
const Retention = reg.RemoveInputReasonRetention
|
||||
|
||||
// removeOnSeat lets reference go: a hold an earlier sweep asked while it was kept is released first (the holder
|
||||
// refuses to remove a held thing; one not held answers released false), then it is removed for retention. why goes with
|
||||
// the release; the hand act records it.
|
||||
func (h *Holder) removeOnSeat(reference, why string) error {
|
||||
if err := h.askSeat(reference, reg.VerbRelease, func(name, digest string) any {
|
||||
return reg.ReleaseInput{Name: name, Digest: digest, Reason: why}
|
||||
}, ®.ReleaseAnswer{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.askSeat(reference, reg.VerbRemove, func(name, digest string) any {
|
||||
return reg.RemoveInput{Name: name, Digest: digest, Reason: Retention}
|
||||
}, ®.RemoveAnswer{})
|
||||
}
|
||||
|
||||
// sweepFinding writes the sweep's finding: what the seat refused, and why.
|
||||
func (h *Holder) sweepFinding(what string, err error) {
|
||||
at := h.now().UTC()
|
||||
f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: sweepFindingKey, Kind: SweepRefused,
|
||||
Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: Module},
|
||||
Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the artifact sweep stopped: a registry seat refused %s (the evidence names why)", what),
|
||||
Headline: "The artifact sweep stopped",
|
||||
Explanation: "The sweep keeps what the mesh still needs and removes the rest through the registry seats. A seat " +
|
||||
"refused one of its asks, so it stopped before anything more was removed; what it kept stays kept.",
|
||||
Needs: "read the seat holder's journal for the refusal; the next sweep asks again.",
|
||||
Resolved: "Resolved: the artifact sweep ran through without a refusal",
|
||||
Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: fmt.Sprintf("%s: %v", what, err)}},
|
||||
Source: "CAT-AS1", Observer: Module, At: at}
|
||||
if werr := h.Bus.Put(FindingsState, sweepFindingKey, f); werr != nil {
|
||||
h.logf("the sweep's finding (%s: %v) could not be written: %v", what, err, werr)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepClean clears the sweep's finding after a sweep no seat refused.
|
||||
func (h *Holder) sweepClean() {
|
||||
if _, found, err := h.Bus.Get(FindingsState, sweepFindingKey); err == nil && found {
|
||||
if err := h.Bus.Delete(FindingsState, sweepFindingKey); err != nil {
|
||||
h.logf("the sweep's finding could not be cleared: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keptWhy is why a kept reference is held, as the seat's hold takes it.
|
||||
func keptWhy(r *Ref) string {
|
||||
if r == nil || len(r.Why) == 0 {
|
||||
return "kept by the artifact sweep"
|
||||
}
|
||||
return "kept: " + strings.Join(r.Why, "; ")
|
||||
}
|
||||
|
||||
// registryForms reads and routes a reference of the registry form, with the two registry seats' names from their contracts
|
||||
// (the registry clients' reference forms, novox/hq ADR 0605 §9, ADR 0634 §4).
|
||||
var registryForms = artifactref.Seats{Image: reg.Seat, Object: ociseat.Seat}
|
||||
@@ -33,7 +33,8 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
@@ -41,6 +42,10 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The builder's keys are read through its published package, taken by its series (ADR 0634 §1).
|
||||
if m.Packages["builds"] != "0.1" {
|
||||
t.Errorf("packages %v: want builds of series 0.1", m.Packages)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"])
|
||||
|
||||
@@ -27,9 +27,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
{
|
||||
Name: "artifact_store_repositories",
|
||||
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
||||
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
||||
"the store's own files; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, each manifest's as its seat says it. " +
|
||||
"A layer two manifests share is counted in each. Read through the registry seats' list (ADR 0632 §4); changes " +
|
||||
"nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
Input: map[string]any{"repository": repoArg},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
v, err := s.View(ctx)
|
||||
@@ -41,9 +41,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "artifact_store_usage",
|
||||
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
||||
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
||||
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
||||
Description: "How large the artifact store is: every manifest's size together, as its seat says it (a layer two " +
|
||||
"manifests share is counted in each), and the largest repositories. Read through the registry seats' list (ADR " +
|
||||
"0632 §4); changes nothing. Replaces du on the store's directory. (r)",
|
||||
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
top, err := count(args, "top", 10, 1, 200)
|
||||
@@ -139,16 +139,6 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// View reads the store now.
|
||||
func (s *Store) View(ctx context.Context) (*View, error) {
|
||||
l, err := s.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, unread := s.Manifests(ctx, l)
|
||||
return &View{L: l, M: m, Unread: unread}, nil
|
||||
}
|
||||
|
||||
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
||||
func unreadNote(v *View) map[string]any {
|
||||
if len(v.Unread) == 0 {
|
||||
|
||||
@@ -4,13 +4,15 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
// View is the store at one moment: its files, and every manifest's content that could be read.
|
||||
// View is the store at one moment, as its seats list it (listed.go).
|
||||
type View struct {
|
||||
L *Listing
|
||||
// M is each manifest read, by Key.
|
||||
M map[string]*Manifest
|
||||
// Unread is each manifest that could not be read, by Key, with why.
|
||||
Unread map[string]string
|
||||
// Held is each manifest a hold on its seat keeps from retention, by Key.
|
||||
Held map[string]bool
|
||||
}
|
||||
|
||||
// Marks are the blobs one manifest keeps from the store's collector: its own content, its
|
||||
|
||||
@@ -1 +1 @@
|
||||
${seat:mesh-artifact-store:reach}
|
||||
${seat:container-image-registry:reach}
|
||||
|
||||
@@ -3,20 +3,36 @@ module artifactstoretools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 // indirect
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -23,5 +21,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
|
||||
// take-in reads text from.
|
||||
const blobBound = 64 << 20
|
||||
|
||||
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
|
||||
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
|
||||
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
|
||||
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
|
||||
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !archive {
|
||||
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
|
||||
}
|
||||
if len(body) > blobBound {
|
||||
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
|
||||
digest, got)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
|
||||
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
|
||||
good := []byte("the archive")
|
||||
sum := sha256.Sum256(good)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
liar := "sha256:" + strings.Repeat("0", 64)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
|
||||
_, _ = w.Write(good)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
ctx := context.Background()
|
||||
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
|
||||
if err != nil || string(body) != string(good) {
|
||||
t.Fatalf("the archive: %q %v", body, err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Fatalf("bytes of another digest are read: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
|
||||
t.Fatalf("a blob the store does not hold: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("a reference outside the store: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||
// collector runs `--dry-run` until this is true.
|
||||
//
|
||||
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||
// holder first, which is what lets the bytes go at the next collection.
|
||||
//
|
||||
// **Nothing a machine reads changes.** The recorded reference stays
|
||||
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||
//
|
||||
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||
// size. No schema change, no second record that could disagree with the store.
|
||||
|
||||
const (
|
||||
// mediaManifest is the type a holder is put and asked for as.
|
||||
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||
// whose only purpose is to name its layer.
|
||||
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
)
|
||||
|
||||
// emptyConfig is the content emptyDigest names.
|
||||
var emptyConfig = []byte("{}")
|
||||
|
||||
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||
var manifestAccept = []string{
|
||||
mediaManifest,
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
type descriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type holderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||
//
|
||||
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||
// nothing here that varies by when or where it was composed.
|
||||
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||
body, err := json.Marshal(holderManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: mediaManifest,
|
||||
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||
})
|
||||
if err != nil {
|
||||
// Marshalling a struct of strings and integers cannot fail.
|
||||
panic(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||
//
|
||||
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||
// blobs published before holders existed (novox/hq issue 253).
|
||||
//
|
||||
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||
// fact the caller reports rather than one this invents a remedy for.
|
||||
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil || !archive {
|
||||
return false, err
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return s.HoldBlob(ctx, repository, digest, size)
|
||||
}
|
||||
|
||||
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||
//
|
||||
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !archive {
|
||||
return true, nil
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
}
|
||||
|
||||
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||
// there already. Answers whether it wrote one.
|
||||
//
|
||||
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||
if s.Address == "" {
|
||||
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||
}
|
||||
body, holder := Holder(digest, size)
|
||||
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if there {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||
// holder mean something.
|
||||
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request.Header.Set("Content-Type", mediaManifest)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||
//
|
||||
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||
//
|
||||
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||
// that is the outcome wanted.
|
||||
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||
if err == Gone {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return repository, digest, kind == "blobs", nil
|
||||
}
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||
return err
|
||||
} else if there {
|
||||
return nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
begun, err := s.client().Do(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||
}
|
||||
where := begun.Header.Get("Location")
|
||||
if where == "" {
|
||||
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||
}
|
||||
if strings.HasPrefix(where, "/") {
|
||||
where = "http://" + s.Address + where
|
||||
}
|
||||
separator := "?"
|
||||
if strings.Contains(where, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
done, err := s.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done.Body.Close()
|
||||
if done.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repository + "@" + digest
|
||||
manifests map[string][]byte // repository + "@" + digest
|
||||
writes []string // every PUT and DELETE, as "METHOD path"
|
||||
}
|
||||
|
||||
func digestOf(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (m *memRegistry) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
m.blobs = map[string][]byte{}
|
||||
m.manifests = map[string][]byte{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.Contains(path, "/blobs/"):
|
||||
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||
key := repository + "@" + digest
|
||||
body, ok := m.blobs[key]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
delete(m.blobs, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
case strings.Contains(path, "/manifests/"):
|
||||
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||
key := repository + "@" + digest
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var named holderManifest
|
||||
if err := json.Unmarshal(body, &named); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m.manifests[key] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodDelete:
|
||||
if _, ok := m.manifests[key]; !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(m.manifests, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
digest := digestOf(body)
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||
// never be mistaken for one of a different blob.
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
one, first := Holder(digest, 42)
|
||||
two, second := Holder(digest, 42)
|
||||
if !bytes.Equal(one, two) || first != second {
|
||||
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||
}
|
||||
if _, other := Holder(digest, 43); other == first {
|
||||
t.Fatal("a different size composed the same holder")
|
||||
}
|
||||
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||
if string(one) != want {
|
||||
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||
}
|
||||
if digestOf(emptyConfig) != emptyDigest {
|
||||
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||
// every sweep, writes one the first time and nothing after.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("a theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
|
||||
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||
}
|
||||
wrote, err := store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("holding a bare archive wrote nothing")
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||
}
|
||||
|
||||
writes := len(m.writes)
|
||||
wrote, err = store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote || len(m.writes) != writes {
|
||||
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An image is its own manifest: nothing is asked.
|
||||
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil || wrote || len(m.writes) != 0 {
|
||||
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||
}
|
||||
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||
}
|
||||
// And a reference that is not the mesh's is refused as such.
|
||||
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||
// A holder left behind would keep the bytes through every collection while the record said
|
||||
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("an old theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
if _, err := store.Hold(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.writes = nil
|
||||
|
||||
if err := store.LetGo(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
want := []string{
|
||||
"DELETE /v2/shell/config/manifests/" + holder,
|
||||
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||
}
|
||||
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(m.manifests) != 0 {
|
||||
t.Fatalf("a holder survived: %v", m.manifests)
|
||||
}
|
||||
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
reference := m.bare("shell/config", []byte("never held"))
|
||||
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.blobs) != 0 {
|
||||
t.Fatalf("the blob survived: %v", m.blobs)
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// An index is let go of with the platform manifests it names, except those a kept index of the same
|
||||
// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in
|
||||
// its own right, and the store's collector keeps every manifest a repository holds — so an index let go
|
||||
// of alone frees nothing of the images it names.
|
||||
|
||||
func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) }
|
||||
|
||||
// indexStore holds indexes and images in one repository, answers GETs with their documents, and records
|
||||
// every delete.
|
||||
type indexStore struct {
|
||||
mu sync.Mutex
|
||||
indexes map[string][]string // digest → the platform manifests it names
|
||||
images map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (s *indexStore) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if children, ok := s.indexes[digest]; ok {
|
||||
var named []string
|
||||
for _, c := range children {
|
||||
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
|
||||
strings.Join(named, ",") + `]}`))
|
||||
return
|
||||
}
|
||||
if s.images[digest] {
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodHead:
|
||||
if _, ok := s.indexes[digest]; ok || s.images[digest] {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodDelete:
|
||||
if _, ok := s.indexes[digest]; !ok && !s.images[digest] {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.deleted = append(s.deleted, digest)
|
||||
delete(s.indexes, digest)
|
||||
delete(s.images, digest)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) {
|
||||
// Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13.
|
||||
s := &indexStore{
|
||||
indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}},
|
||||
images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true},
|
||||
}
|
||||
store := s.serve(t)
|
||||
repository := "upstream/docker.io/library/golang"
|
||||
old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1)
|
||||
kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2)
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
|
||||
if err := store.LetGo(context.Background(), old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its own platform first, the index last; the platform the kept index names is left in place.
|
||||
if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) {
|
||||
t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted)
|
||||
}
|
||||
if !s.images[digestN(12)] || !s.images[digestN(13)] {
|
||||
t.Fatal("a platform a kept index names was let go of")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}}
|
||||
store := s.serve(t)
|
||||
// No Spare: as before ADR 0257, the index alone.
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(1)}) {
|
||||
t.Fatalf("without a Spare, deleted %v", s.deleted)
|
||||
}
|
||||
// An image is one delete, with a Spare or without.
|
||||
s.deleted = nil
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(5)}) {
|
||||
t.Fatalf("an image was let go of as %v", s.deleted)
|
||||
}
|
||||
// An index the store no longer holds is Gone, and nothing is deleted on its account.
|
||||
s.deleted = nil
|
||||
err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9))
|
||||
if !errors.Is(err, Gone) || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
store := s.serve(t)
|
||||
store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") }
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1))
|
||||
if err == nil || !strings.Contains(err.Error(), "was kept") {
|
||||
t.Fatalf("a spare that could not be read: %v", err)
|
||||
}
|
||||
if len(s.deleted) != 0 {
|
||||
t.Fatalf("deleted %v without knowing what a kept index names", s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The spare list is read whole before the sweep: a kept index that cannot be read is an error before
|
||||
// anything is deleted, and a repository not read before keeps its index.
|
||||
func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
store := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2)
|
||||
old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1)
|
||||
if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil {
|
||||
t.Fatal("a kept index the store would not answer for was taken as naming nothing")
|
||||
}
|
||||
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
good := s.serve(t)
|
||||
spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
good.Spare = spare
|
||||
if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
@@ -1,325 +0,0 @@
|
||||
// Package registry speaks to the mesh's artifact store over its own door.
|
||||
//
|
||||
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||
//
|
||||
// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets
|
||||
// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease.
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Store is the artifact store at an address, as this machine reaches it.
|
||||
type Store struct {
|
||||
// Address is `host:port` — the store as the caller reaches it now, composed and never
|
||||
// recorded (novox/hq 04-ISSUES/102).
|
||||
Address string
|
||||
// HTTP is the client used; nil is a client with a modest timeout.
|
||||
HTTP *http.Client
|
||||
// Spare says, for one repository, which manifests a kept index there names: the platform
|
||||
// manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every
|
||||
// platform manifest — an index is then let go of alone, as it was before.
|
||||
Spare func(ctx context.Context, repository string) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
|
||||
// the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
|
||||
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
|
||||
// error" is how a cautious loop became one that did nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// LetGo asks the store to drop one artifact the mesh recorded making.
|
||||
//
|
||||
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
|
||||
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||
// and composes the address here at the moment of use.
|
||||
//
|
||||
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||
// issue 253); an image in one.
|
||||
//
|
||||
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||
// which of the two happened.
|
||||
func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
|
||||
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
|
||||
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
|
||||
// where the provenance is known, which is the sweep reading its own build records, not here
|
||||
// where the only job is to refuse anything that is not plainly ours.
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
|
||||
// delete for a reference of unknown shape is how a sweep reaches something that is not
|
||||
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kind == "blobs" {
|
||||
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||
// naming the blob would keep its bytes through every collection while the record said
|
||||
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if kind == "manifests" && s.Spare != nil {
|
||||
// **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put
|
||||
// under its own digest, so each is a manifest of the repository in its own right, and the
|
||||
// store's collector keeps every manifest a repository holds: an index let go of alone frees
|
||||
// no byte of the images it names. Before, not after, so a sweep cut short leaves the index
|
||||
// to be asked about again, and with it the list of what is still to go.
|
||||
if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||
}
|
||||
|
||||
// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same
|
||||
// repository also names. A manifest that is not an index names none.
|
||||
func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error {
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if err != nil || len(children) == 0 {
|
||||
// Gone: there is no index to read, and the delete that follows says so.
|
||||
if errors.Is(err, Gone) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
spared, err := s.Spare(ctx, repository)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w",
|
||||
repository, reference, err)
|
||||
}
|
||||
for _, child := range children {
|
||||
if spared[child] {
|
||||
continue
|
||||
}
|
||||
if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// anyManifest is every manifest the store may hold, indexes included, so it answers with the
|
||||
// document as it is.
|
||||
var anyManifest = []string{
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
// Platforms is the manifests an index names, by digest; none for a manifest that is not an index.
|
||||
// Gone when the store does not hold it.
|
||||
func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("Accept", strings.Join(anyManifest, ", "))
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest)
|
||||
}
|
||||
var document struct {
|
||||
Manifests []struct {
|
||||
Digest string `json:"digest"`
|
||||
} `json:"manifests"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil {
|
||||
return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
|
||||
}
|
||||
var out []string
|
||||
for _, m := range document.Manifests {
|
||||
if strings.HasPrefix(m.Digest, "sha256:") {
|
||||
out = append(out, m.Digest)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an
|
||||
// image reference of `touching` is in, each kept manifest there and every platform manifest a kept
|
||||
// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not
|
||||
// read here is answered with an error, so an index in it is kept. A kept index the store no longer
|
||||
// holds names nothing.
|
||||
func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) {
|
||||
imageIn := func(reference string) (repository, digest string, ok bool) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", "", false
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil || kind != "manifests" {
|
||||
return "", "", false
|
||||
}
|
||||
return repository, digest, true
|
||||
}
|
||||
read := map[string]map[string]bool{}
|
||||
for _, reference := range touching {
|
||||
if repository, _, ok := imageIn(reference); ok {
|
||||
read[repository] = map[string]bool{}
|
||||
}
|
||||
}
|
||||
for _, reference := range kept {
|
||||
repository, digest, ok := imageIn(reference)
|
||||
spared, touched := read[repository]
|
||||
if !ok || !touched {
|
||||
continue
|
||||
}
|
||||
spared[digest] = true
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if errors.Is(err, Gone) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err)
|
||||
}
|
||||
for _, c := range children {
|
||||
spared[c] = true
|
||||
}
|
||||
}
|
||||
return func(_ context.Context, repository string) (map[string]bool, error) {
|
||||
spared, done := read[repository]
|
||||
if !done {
|
||||
return nil, fmt.Errorf("%s was not read before the sweep began", repository)
|
||||
}
|
||||
return spared, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
|
||||
return nil
|
||||
case http.StatusNotFound:
|
||||
return Gone
|
||||
case http.StatusMethodNotAllowed:
|
||||
// The registry was started without deletion enabled. Said plainly, because the remedy is
|
||||
// a setting on the store's module and not anything about this artifact.
|
||||
return fmt.Errorf(
|
||||
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||
default:
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||
}
|
||||
}
|
||||
|
||||
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
|
||||
//
|
||||
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
|
||||
// `<repository>/blobs/sha256:<hex>` is a blob.
|
||||
func split(path string) (repository, kind, digest string, err error) {
|
||||
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
|
||||
return before, "manifests", "sha256:" + after, nil
|
||||
}
|
||||
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
|
||||
return before, "blobs", "sha256:" + after, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
|
||||
}
|
||||
|
||||
// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A
|
||||
// blob, or a manifest that names none, is not. Gone when the store does not hold it.
|
||||
func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil || kind != "manifests" {
|
||||
return false, err
|
||||
}
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
return len(children) > 0, err
|
||||
}
|
||||
|
||||
// HoldsManifest is whether the store holds the manifest a recorded image reference names.
|
||||
func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if kind != "manifests" {
|
||||
return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference)
|
||||
}
|
||||
return s.has(ctx, s.url(repository, kind, digest), anyManifest...)
|
||||
}
|
||||
|
||||
// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image.
|
||||
// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that
|
||||
// are not there).
|
||||
func (s Store) Holds(ctx context.Context, reference string) error {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
if kind == "blobs" {
|
||||
_, err := s.blobSize(ctx, repository, digest)
|
||||
return err
|
||||
}
|
||||
held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
return Gone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
|
||||
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
|
||||
|
||||
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||
// that does not have the thing does not have its blob either.
|
||||
if answer == http.StatusNotFound {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", "7")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||
}
|
||||
asked = append(asked, r.URL.Path)
|
||||
w.WriteHeader(answer)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
|
||||
// different endpoints, and asking at the wrong one answers 404 — which this would then
|
||||
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
ctx := context.Background()
|
||||
|
||||
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
|
||||
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
|
||||
if err := store.LetGo(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||
_, holder := Holder("sha256:def456", 7)
|
||||
want := []string{
|
||||
"/v2/web/app/manifests/sha256:abc123",
|
||||
"/v2/web/config/manifests/" + holder,
|
||||
"/v2/web/config/blobs/sha256:def456",
|
||||
}
|
||||
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
|
||||
// The outcome wanted, already true. Told apart from success only so the sweep can say which
|
||||
// happened; both are recorded, because retrying for ever is the thing to avoid.
|
||||
store, _ := fakeStore(t, http.StatusNotFound)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
|
||||
// The registry answers 405 when it was started without deletion enabled. The remedy is a
|
||||
// setting on the store's module, and saying "405" would send somebody to the wrong place.
|
||||
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if err == nil {
|
||||
t.Fatal("a store that refuses deletion was read as success")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
|
||||
t.Fatalf("the refusal does not name the remedy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
|
||||
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
|
||||
// A reference of another shape — a vendor's image, a package version — is refused rather
|
||||
// than composed into a delete somewhere that is not the mesh's store.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"registry@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
if err := store.LetGo(context.Background(), reference); err == nil {
|
||||
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
|
||||
// record and not about the store (novox/hq issue 226).
|
||||
//
|
||||
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
|
||||
// first live run met a reference recorded with the store's old address, read the refusal as "the
|
||||
// store refuses everything", and collected none of the 1681 it had found.
|
||||
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
err := store.LetGo(context.Background(), reference)
|
||||
if !errors.Is(err, ErrNotOurs) {
|
||||
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "artifact-store-tools",
|
||||
"version": "1",
|
||||
"packages": {
|
||||
"builds": "0.1"
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-artifact-sweep",
|
||||
@@ -14,21 +17,30 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"artifact-store"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
"seat:mesh-catalog.modules",
|
||||
"seat:mesh-catalog.module",
|
||||
"mesh-delivery-assigner.assigner_read_wants",
|
||||
"mesh-delivery-indexer.indexer_read_manifests",
|
||||
"seat:oci-artifact-registry.hold",
|
||||
"seat:oci-artifact-registry.release",
|
||||
"seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold",
|
||||
"seat:container-image-registry.release",
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.list",
|
||||
"seat:oci-artifact-registry.list"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"node-engine.have"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-assigner.views.wants",
|
||||
"mesh-delivery-indexer.views.manifests"
|
||||
],
|
||||
"state": [
|
||||
"findings"
|
||||
],
|
||||
@@ -36,7 +48,9 @@
|
||||
"artifact_store_repositories",
|
||||
"artifact_store_usage",
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect"
|
||||
"artifact_store_collect",
|
||||
"artifact_store_first_views",
|
||||
"artifact_store_first_view_again"
|
||||
],
|
||||
"config": {
|
||||
"older-store-names": {
|
||||
@@ -84,7 +98,6 @@
|
||||
"artifact-store-tools"
|
||||
],
|
||||
"env": {
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
|
||||
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names",
|
||||
"MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address"
|
||||
}
|
||||
|
||||
@@ -2,10 +2,10 @@ module asuszephyrusg14
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module avahi
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module blueman
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module bluetooth
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -3,20 +3,20 @@ module brother
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
+8
-10
@@ -1,15 +1,13 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -23,5 +21,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -122,7 +122,7 @@ The agent kept reaching for `ssh <machine> journalctl` while the service manager
|
||||
- **The "instead of" table.** Every seat verb and module tool may say which shell commands it replaces
|
||||
(`replaces`, in the seat's definition or the module's manifest). The module reads the bus — the seats' verbs from the
|
||||
seats' contract, the placed modules' manifests from `mesh-delivery-indexer.modules` and `.manifests`, the
|
||||
machines and where each module runs from `mesh-delivery-assigner.placement` (issue 1126: the controller that
|
||||
machines and where each module runs from `mesh-delivery-assigner.assignments` (issue 1126: the controller that
|
||||
answered this is not placed) — at start, every ten minutes and on `claude_code_render`, keeps it in its
|
||||
state (`mesh-tools.json`), and renders it into `CLAUDE.md` as one row per seat or module: `<node>/<seat>.` and
|
||||
each verb with the commands it replaces. Generated, never written by hand: a verb that gains `replaces` is in
|
||||
@@ -206,10 +206,20 @@ judges the record as this module would have made it, composes the ask again from
|
||||
the text's digest, in time), and only then writes the section into its own `approved.json`, which the managed
|
||||
instruction file renders under "Instructions the operator approved". A proposal is decided once per node, and an approval older than the last change applied to its section is never applied; a record
|
||||
deleted from the bus removes nothing. Refused: shell lines, front matter, any `@` after a non-letter (a file the instruction file may read in), invisible or control characters, headings of level one to three, anything shaped like
|
||||
a secret, more than 16 KiB, a name the terminal sets, and — at once, naming the terminal's line — whenever no channel
|
||||
a secret, more than 16 KiB, and — at once, naming the terminal's line — whenever no channel
|
||||
that proves its sender is there. The controller's hand-act log records each warrant once (`warranted`), read from the
|
||||
router's record. The text is never logged.
|
||||
|
||||
**A section registered before is replaced or removed the same way** (hq issue 340, note 14): a section registered for the
|
||||
mesh or the node before the controller's terminal took that over, or set there, is proposed over by name. The proposal
|
||||
carries `over`, the digest of the registered text, which the act the operator's answer binds names; the ask says
|
||||
plainly that it replaces (or removes) the registered section, with the registered text's fingerprint, and shows the new
|
||||
text as any other. On Approve each node writes the section into `approved.json` with that `over`, and the managed
|
||||
instruction file renders it in the place of the registered section (a removal leaves the section out); on Decline or at
|
||||
expiry nothing changes. A replacement keeps the registered section's own heading form, because it replaces a mesh
|
||||
section; the heading rule stays for every new shared section. `claude_code_proposals` shows each proposal's `registered
|
||||
section` and `claude_code_registrations` lists what approvals replaced or removed.
|
||||
|
||||
**A session asks the operator a question on Telegram** (hq ADR 0332, issue 589). An agent that needs a decision while
|
||||
the operator may be away from the terminal calls `claude_code_ask` with a headline, the context in plain words, two to
|
||||
four options (a label and what choosing it means, exactly one recommended, with why), what it does if nobody
|
||||
|
||||
@@ -637,6 +637,14 @@ func mergeSettings(a, b map[string]any) map[string]any {
|
||||
// registered for the mesh, then those for this node.
|
||||
func InstructionSections(c Config) string {
|
||||
var b strings.Builder
|
||||
// The approved sections that replace or remove a registered one, by its name and the digest of its text, and the
|
||||
// proposals already placed where the registered section stood.
|
||||
over, placed := map[string]ApprovedSection{}, map[string]bool{}
|
||||
for _, s := range c.Approved {
|
||||
if s.Over != "" {
|
||||
over[s.Name+"|"+s.Over] = s
|
||||
}
|
||||
}
|
||||
registered := "Registered through the `claude-code` module before that moved to the controller's terminal; change them there, in the module's `instructions` setting."
|
||||
for _, part := range []struct {
|
||||
title, note string
|
||||
@@ -654,14 +662,31 @@ func InstructionSections(c Config) string {
|
||||
}
|
||||
fmt.Fprintf(&b, "\n## %s\n\n%s\n", part.title, part.note)
|
||||
for _, it := range sections {
|
||||
fmt.Fprintf(&b, "\n### %s\n\n%s\n", it.Name, strings.TrimSpace(it.Files[it.Name+".md"]))
|
||||
text := it.Files[it.Name+".md"]
|
||||
if s, ok := over[it.Name+"|"+textDigest(text)]; ok {
|
||||
// What the operator approved over this registered section stands in its place, or leaves it out.
|
||||
placed[s.Proposal] = true
|
||||
if s.Removes {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "\n### %s\n\n*Replaced on the operator's approval (proposed from %s; approved %s).*\n\n%s\n", it.Name, s.From,
|
||||
s.Approved.UTC().Format("2006-01-02 15:04 UTC"), strings.TrimSpace(s.Text))
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "\n### %s\n\n%s\n", it.Name, strings.TrimSpace(text))
|
||||
}
|
||||
}
|
||||
if len(c.Approved) > 0 {
|
||||
var approved []ApprovedSection
|
||||
for _, s := range c.Approved {
|
||||
if !s.Removes && !placed[s.Proposal] {
|
||||
approved = append(approved, s)
|
||||
}
|
||||
}
|
||||
if len(approved) > 0 {
|
||||
b.WriteString("\n## Instructions the operator approved\n\nProposed by an agent through the `claude-code` module and " +
|
||||
"approved by the operator on a channel that proves who answered (novox/hq ADR 0274). Changed or removed only the " +
|
||||
"same way: `claude_code_instructions_propose`.\n")
|
||||
for _, s := range c.Approved {
|
||||
for _, s := range approved {
|
||||
fmt.Fprintf(&b, "\n### %s\n\n*Proposed from %s; approved %s.*\n\n%s\n", s.Name, s.From,
|
||||
s.Approved.UTC().Format("2006-01-02 15:04 UTC"), strings.TrimSpace(s.Text))
|
||||
}
|
||||
|
||||
@@ -269,12 +269,18 @@ func TestTheManifestListsEveryToolServed(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
if !slices.Equal(m.Invokes, []string{"seat:mesh-hand-acts.warranted"}) {
|
||||
t.Errorf("invokes = %v, want the hand-act seat's warranted alone", m.Invokes)
|
||||
// The hand-act seat's warranted and hand-act, the seat mesh-catalog's seat list (ADR 0635 §1), and the owners' read tools of the three followed states; no verb of
|
||||
// the controller, and no read of the indexer's or the assigner's state (ADR 0615 §1): their views are consumed.
|
||||
if !slices.Equal(m.Invokes, []string{"seat:mesh-hand-acts.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.seat-definitions", ModulesTool, ManifestsTool, AssignmentsTool}) ||
|
||||
!slices.Equal(m.Reads, []string{"claude-licence-manager.bindings"}) ||
|
||||
!slices.Equal(m.Consumes, []string{"mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests", "mesh-delivery-assigner.views.assignments", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"}) {
|
||||
t.Errorf("invokes = %v, reads = %v, consumes = %v", m.Invokes, m.Reads, m.Consumes)
|
||||
}
|
||||
if src, _ := os.ReadFile("proposals.go"); !strings.Contains(string(src), `"seat:mesh-hand-acts.warranted"`) ||
|
||||
strings.Contains(string(src), "seat:mesh-controller.warranted") {
|
||||
@@ -285,6 +291,7 @@ func TestTheManifestListsEveryToolServed(t *testing.T) {
|
||||
listed[n] = true
|
||||
}
|
||||
p, _ := node(t, "laptop")
|
||||
meshViews = newMeshViews()
|
||||
served := tools(p, nil, NewServerView(p), memConfig{}, NewConfigView(p))
|
||||
for _, tool := range served {
|
||||
if !listed[tool.Name] {
|
||||
|
||||
@@ -25,7 +25,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// ForgeProvision is the provision whose credential this is, and the file this module's manifest has it delivered to.
|
||||
@@ -55,16 +56,16 @@ func (f *ForgeHealth) key() string { return "claude-code.machine." + f.Node + ".
|
||||
|
||||
// Look is the condition to say after one placing of the credential, nil when nothing changed: raised on a failure, again
|
||||
// when its reason changes, cleared when it places again; on the first look a success clears what a run before raised.
|
||||
func (f *ForgeHealth) Look(err error) *asks.Condition {
|
||||
func (f *ForgeHealth) Look(err error) *oc.Condition {
|
||||
first := !f.looked
|
||||
f.looked = true
|
||||
subject := asks.Scope{Scope: "machine", ID: f.Node, Machine: f.Node}
|
||||
subject := oc.ConditionSubject{Scope: "machine", ID: f.Node, Machine: f.Node}
|
||||
if err == nil {
|
||||
if f.raised == "" && !first {
|
||||
return nil
|
||||
}
|
||||
f.raised = ""
|
||||
return &asks.Condition{Key: f.key(), Subject: subject, State: asks.StateCleared}
|
||||
return &oc.Condition{Key: f.key(), Subject: subject, State: occ.StateCleared}
|
||||
}
|
||||
why := err.Error()
|
||||
if why == f.raised {
|
||||
@@ -72,10 +73,10 @@ func (f *ForgeHealth) Look(err error) *asks.Condition {
|
||||
}
|
||||
f.raised = why
|
||||
needs := "read claude-code's journal on " + f.Node + " and the owner of its forge credential file"
|
||||
return &asks.Condition{Key: f.key(), Severity: asks.SeverityWarning, State: asks.StateRaised, Subject: subject,
|
||||
return &oc.Condition{Key: f.key(), Severity: occ.SeverityWarning, State: occ.StateRaised, Subject: subject,
|
||||
Headline: "Agents on " + f.Node + " cannot use the forge credential",
|
||||
Summary: why, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " +
|
||||
Explanation: occ.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " +
|
||||
"clone or fetch the mesh's private repositories.",
|
||||
Resolved: "The forge's read credential is placed again on " + f.Node + "."}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
)
|
||||
|
||||
const agentToken = "0123456789abcdef0123456789abcdef01234567"
|
||||
@@ -316,7 +316,7 @@ func TestTheForgeCredentialIsTheRuntimesAndItsFailureIsAConditionOfTheModule(t *
|
||||
got := ""
|
||||
if c != nil {
|
||||
got = c.State
|
||||
if _, err := asks.EncodeCondition(*c); err != nil {
|
||||
if _, err := occ.EncodeCondition(*c); err != nil {
|
||||
t.Errorf("look %d: the condition is refused: %v", i, err)
|
||||
}
|
||||
if c.Key != "claude-code.machine.anchor.forge-credential" {
|
||||
|
||||
@@ -9,25 +9,25 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// meshForTheGuard is a mesh of two machines and the verbs that replace what an agent runs over ssh, as the
|
||||
// seats' contract and the modules' manifests say them.
|
||||
func meshForTheGuard() GuardData {
|
||||
seats := []catalogue.Seat{
|
||||
{Name: "mesh-placement", Scope: catalogue.ScopeMesh, Serves: []catalogue.Verb{
|
||||
{Name: "placement", Description: "Read placement: one node's key.", Replaces: []string{"hostnamectl", "uptime"}}}},
|
||||
{Name: "node-service-manager", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||
defined := []seatdef.Definition{
|
||||
{Seat: "mesh-module-assigner", Scope: "mesh", Verbs: []seatdef.Verb{
|
||||
{Name: "assignments", Description: "Read assignments: one node's key.", Replaces: []string{"hostnamectl", "uptime"}}}},
|
||||
{Seat: "node-service-manager", Scope: "node", Verbs: []seatdef.Verb{
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states.", Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "restart", Description: "Restart one unit.", Replaces: []string{"systemctl restart"}},
|
||||
{Name: "journal", Description: "The last lines of one unit's journal.", Replaces: []string{"journalctl"}}}},
|
||||
{Name: "node-hostname", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||
{Seat: "node-hostname", Scope: "node", Verbs: []seatdef.Verb{
|
||||
{Name: "add", Description: "Add one address and its names to the operator's lines of /etc/hosts.", Replaces: []string{"edit /etc/hosts", "HOSTALIASES"}}}},
|
||||
}
|
||||
modules := []ModuleTools{{Module: "docker", On: []string{"anchor"}, Replaces: map[string][]string{"docker_logs": {"docker logs"}, "docker_list": {"docker ps"}}},
|
||||
{Module: "systemd", On: []string{"anchor", "laptop"}}}
|
||||
return GuardData{Node: "laptop", Replacements: ReplacementsOf(seats, modules), Machines: []Machine{
|
||||
return GuardData{Node: "laptop", Replacements: ReplacementsOf(defined, modules), Machines: []Machine{
|
||||
{Name: "anchor", Domains: []string{"anchor" + InternalSuffix, "anchor.example"}, Addresses: []string{"192.0.2.10", "10.10.0.1"}},
|
||||
{Name: "laptop", Domains: []string{"laptop" + InternalSuffix}, Addresses: []string{"10.10.0.4"}},
|
||||
}}
|
||||
@@ -80,14 +80,14 @@ func TestSSHToAMeshMachineIsRefusedNamingTheTool(t *testing.T) {
|
||||
"ssh -o StrictHostKeyChecking=no 10.10.0.1 docker ps -a": "anchor/docker.docker_list",
|
||||
"ssh anchor docker logs --tail 100 nats": "anchor/docker.docker_logs",
|
||||
"ssh laptop docker logs x": "anchor/docker.docker_logs",
|
||||
"ssh a uptime": "mesh-placement.placement",
|
||||
"ssh a uptime": "mesh-module-assigner.assignments",
|
||||
"cd /tmp && ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"sudo -u root ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"timeout 10 ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"env LANG=C ssh anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"bash -c 'ssh anchor journalctl -u x'": "anchor/node-service-manager.journal",
|
||||
`echo "$(ssh anchor journalctl -n 5)" | tail`: "anchor/node-service-manager.journal",
|
||||
"for n in anchor laptop; do ssh $n uptime; done; ssh anchor uptime": "mesh-placement.placement",
|
||||
"for n in anchor laptop; do ssh $n uptime; done; ssh anchor uptime": "mesh-module-assigner.assignments",
|
||||
"ssh -J anchor 203.0.113.9 journalctl": "anchor/node-service-manager.journal",
|
||||
"ssh via-anchor journalctl": "anchor/node-service-manager.journal",
|
||||
"ssh nas.lan journalctl": "anchor/node-service-manager.journal",
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// handActs records the hand acts of the views reader through the seat mesh-hand-acts's client. An act the seat does not
|
||||
// take is an error, and the reader then does nothing (the views package's rule).
|
||||
type handActs struct{}
|
||||
|
||||
func (handActs) RecordHandAct(a views.HandAct) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(meshhandactsclient.Request{What: a.What, Why: a.Why, Cause: a.Cause, Verb: a.Verb})
|
||||
return err
|
||||
}
|
||||
@@ -1,16 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mesh "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// aBus is the states claude-code reads, as a test holds them: by read, key and value.
|
||||
@@ -33,33 +42,33 @@ func (b aBus) Get(read, key string) ([]byte, bool, error) {
|
||||
}
|
||||
|
||||
// someSeats are two seats as the seats' contract says them: one held once for the mesh, one on every machine.
|
||||
func someSeats() []catalogue.Seat {
|
||||
return []catalogue.Seat{
|
||||
{Name: "mesh-placement", Scope: catalogue.ScopeMesh, Serves: []catalogue.Verb{
|
||||
{Name: "placement", Description: "Read placement: one node's key.", Replaces: []string{"hostnamectl", "uptime"}},
|
||||
func someSeats() []seatdef.Definition {
|
||||
return []seatdef.Definition{
|
||||
{Seat: "mesh-module-assigner", Domain: "Core", Scope: "mesh", Verbs: []seatdef.Verb{
|
||||
{Name: "assignments", Description: "Read assignments: one node's key.", Replaces: []string{"hostnamectl", "uptime"}},
|
||||
{Name: "propose", Description: "Propose a change."}}},
|
||||
{Name: "node-service-manager", Scope: catalogue.ScopeNode, Serves: []catalogue.Verb{
|
||||
{Seat: "node-service-manager", Domain: "Core", Scope: "node", Verbs: []seatdef.Verb{
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot.", Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "journal", Description: "The last lines of one unit's journal.", Replaces: []string{"journalctl"}}}},
|
||||
}
|
||||
}
|
||||
|
||||
// aMesh is two machines, anchor (with a public domain) and laptop, docker and zsh placed on anchor, docker's manifest
|
||||
// saying what its tool replaces, as the seat mesh-placement and the indexer keep them on the bus.
|
||||
// saying what its tool replaces, as the seat mesh-module-assigner and the indexer keep them on the bus.
|
||||
func aMesh(t *testing.T) aBus {
|
||||
t.Helper()
|
||||
b := aBus{PlacementRead: {}, ModulesRead: {}, ManifestsRead: {}}
|
||||
for _, p := range []delivery.Placement{
|
||||
{V: 1, Node: "anchor", Decided: delivery.NodeDecided{PublicDomain: "Anchor.Example"},
|
||||
Modules: []delivery.PlacedModule{{Module: "docker"}, {Module: "zsh"}}},
|
||||
b := aBus{AssignmentsRead: {}, ModulesRead: {}, ManifestsRead: {}}
|
||||
for _, p := range []mesh.Assignments{
|
||||
{V: 1, Node: "anchor", Decided: mesh.NodeDecided{PublicDomain: "Anchor.Example"},
|
||||
Modules: []mesh.PlacedModule{{Module: "docker"}, {Module: "zsh"}}},
|
||||
{V: 1, Node: "laptop"},
|
||||
} {
|
||||
values, err := delivery.PlacementValues(p)
|
||||
values, err := assignerclient.AssignmentsValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, v := range values {
|
||||
b[PlacementRead][k] = v
|
||||
b[AssignmentsRead][k] = v
|
||||
}
|
||||
}
|
||||
for module, manifest := range map[string]string{
|
||||
@@ -67,14 +76,14 @@ func aMesh(t *testing.T) aBus {
|
||||
"zsh": `{"module":"zsh"}`,
|
||||
} {
|
||||
digest := fmt.Sprintf("%x", sha256.Sum256([]byte(manifest)))
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
raw, err := json.Marshal(delivery.Manifest{V: 1, Module: module, JSON: manifest})
|
||||
mkey, _ := catalogclient.ManifestsKey(digest)
|
||||
raw, err := json.Marshal(catalog.ManifestsByDigestValue{V: 1, Module: module, Manifest: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b[ManifestsRead][mkey] = raw
|
||||
key, _ := delivery.ModulesKey(module)
|
||||
b[ModulesRead][key], _ = json.Marshal(delivery.Module{V: 1, None: "nothing is built of it", Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
key, _ := catalogclient.ModulesKey(module)
|
||||
b[ModulesRead][key], _ = json.Marshal(catalog.ModulesByModuleValue{V: 1, None: "nothing is built of it", Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -96,9 +105,43 @@ func TestTheMeshsToolsAreReadFromTheBus(t *testing.T) {
|
||||
for _, r := range m.Replacements {
|
||||
addresses = append(addresses, r.Address)
|
||||
}
|
||||
if got := strings.Join(addresses, ","); got != "mesh-placement.placement,node-service-manager.status,node-service-manager.journal,docker.docker_logs" {
|
||||
if got := strings.Join(addresses, ","); got != "mesh-module-assigner.assignments,node-service-manager.status,node-service-manager.journal,docker.docker_logs" {
|
||||
t.Errorf("replacements: %s", got)
|
||||
}
|
||||
t.Run("the again tool answers the operator and the healers only", func(t *testing.T) {
|
||||
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.mesh-healers": false, "shanks.mesh-config": true} {
|
||||
if err := seatlist.Refusal("claude_code_seat_list_again", seatListAgainCallers, caller); (err != nil) != refused {
|
||||
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("the seats are the seat list read from the bus", func(t *testing.T) {
|
||||
// A table made from what the seat mesh-catalog answers (ADR 0635 §1): no compiled list, so before the reader
|
||||
// has started nothing replaces a command, and after it the seats' verbs are the seat's own.
|
||||
seatListMu.Lock()
|
||||
seatList = nil
|
||||
seatListMu.Unlock()
|
||||
if got := definedSeats(); len(got) != 0 {
|
||||
t.Fatalf("seats before the reader started: %v", got)
|
||||
}
|
||||
startSeatList(listBus{"mesh-module-assigner": someSeats()[0], "node-service-manager": someSeats()[1]})
|
||||
defer func() { seatListMu.Lock(); seatList = nil; seatListMu.Unlock() }()
|
||||
// The reader's first read ends when Ready closes (no sleeping).
|
||||
select {
|
||||
case <-seatList.Ready():
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("the seat list was not read: %+v", seatList.Status())
|
||||
}
|
||||
got, err := ReadMeshToolsFrom(aMesh(t), definedSeats(), resolving)
|
||||
var addrs []string
|
||||
for _, r := range got.Replacements {
|
||||
addrs = append(addrs, r.Address)
|
||||
}
|
||||
// The foundation's bootstrapped seats are held beside the two the seat answered, in reading order.
|
||||
if err != nil || !slices.Contains(addrs, "mesh-module-assigner.assignments") || !slices.Contains(addrs, "node-service-manager.journal") {
|
||||
t.Fatalf("replacements from the bus: %v, %v", addrs, err)
|
||||
}
|
||||
})
|
||||
if len(m.Machines) != 2 || strings.Join(m.Machines[0].Domains, ",") != "anchor.internal,anchor.example" ||
|
||||
strings.Join(m.Machines[0].Addresses, ",") != "10.10.0.1,192.0.2.10" || strings.Join(m.Machines[1].Addresses, ",") != "10.10.0.4" {
|
||||
t.Errorf("machines: %+v", m.Machines)
|
||||
@@ -116,7 +159,7 @@ func TestTheInsteadOfTableIsGeneratedFromTheRecords(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"| `<node>/node-service-manager.` | `status` — `systemctl status`, `systemctl is-active` · `journal` — `journalctl` |",
|
||||
"| `<node>/docker.` | `docker_logs` — `docker logs` |",
|
||||
"| `mesh-placement.` | `placement` — `hostnamectl`, `uptime` |",
|
||||
"| `mesh-module-assigner.` | `assignments` — `hostnamectl`, `uptime` |",
|
||||
"Never `ssh` to a mesh machine",
|
||||
} {
|
||||
if !strings.Contains(table, want) {
|
||||
@@ -124,14 +167,14 @@ func TestTheInsteadOfTableIsGeneratedFromTheRecords(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if strings.Index(table, "node-service-manager.") > strings.Index(table, "docker.") ||
|
||||
strings.Index(table, "docker.") > strings.Index(table, "`mesh-placement.`") {
|
||||
strings.Index(table, "docker.") > strings.Index(table, "`mesh-module-assigner.`") {
|
||||
t.Errorf("not the machines' seats, then modules, then the mesh's:\n%s", table)
|
||||
}
|
||||
// What the guard here named most comes first: its row, and in its row its verb.
|
||||
m.Refused = map[string]int{"node-service-manager.journal": 3, "mesh-placement.placement": 1}
|
||||
m.Refused = map[string]int{"node-service-manager.journal": 3, "mesh-module-assigner.assignments": 1}
|
||||
ranked := InsteadOf(&m)
|
||||
if !strings.Contains(ranked, "| `<node>/node-service-manager.` | `journal` — `journalctl` · `status`") ||
|
||||
strings.Index(ranked, "`mesh-placement.`") > strings.Index(ranked, "`<node>/docker.`") {
|
||||
strings.Index(ranked, "`mesh-module-assigner.`") > strings.Index(ranked, "`<node>/docker.`") {
|
||||
t.Errorf("not ranked by the guard's record:\n%s", ranked)
|
||||
}
|
||||
m.Refused = nil
|
||||
@@ -178,8 +221,8 @@ func TestTheMeshsToolsAreKeptOnlyWhenTheyChange(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Where a module runs is read from the seat mesh-placement's placement (issue 1126 B13), never the controller's answer.
|
||||
func TestTheMachinesRunningAModuleAreReadFromPlacement(t *testing.T) {
|
||||
// Where a module runs is read from the seat mesh-module-assigner's assignments (issue 1126 B13), never the controller's answer.
|
||||
func TestTheMachinesRunningAModuleAreReadFromAssignments(t *testing.T) {
|
||||
b := aMesh(t)
|
||||
if got, err := NodesRunning(b, "zsh"); err != nil || strings.Join(got, ",") != "anchor" {
|
||||
t.Errorf("zsh runs on %v (%v)", got, err)
|
||||
@@ -188,14 +231,48 @@ func TestTheMachinesRunningAModuleAreReadFromPlacement(t *testing.T) {
|
||||
t.Errorf("a module that runs nowhere: %v (%v)", got, err)
|
||||
}
|
||||
if _, err := NodesRunning(aBus{}, "zsh"); err == nil {
|
||||
t.Error("an unreadable placement was read as no machine")
|
||||
t.Error("an unreadable assignments was read as no machine")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0615 §1: the tools and the assignments come from the owners' first views and their views events, never their state.
|
||||
// A reader whose first views are answered from the fixture reads what the state read gave, and a views event changes
|
||||
// what it reads (CAT-CC3).
|
||||
func TestTheMeshIsReadFromTheOwnersFirstViews(t *testing.T) {
|
||||
bus := aMesh(t)
|
||||
r := newMeshViews()
|
||||
for _, name := range []string{ModulesRead, ManifestsRead, AssignmentsRead} {
|
||||
state := strings.TrimPrefix(name, strings.Split(name, ".")[0]+".")
|
||||
values := map[string]json.RawMessage{}
|
||||
for k, v := range bus[name] {
|
||||
values[k] = v
|
||||
}
|
||||
f := r.Follow(strings.Split(name, ".")[0], state, "x")
|
||||
f.Ask = func(context.Context, string) (views.Answer, error) {
|
||||
return views.Answer{V: views.Version, Owner: f.Owner, State: state, Revision: 5, Created: time.Unix(1, 0), Values: values, At: time.Now()}, nil
|
||||
}
|
||||
if err := f.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, err := ReadMeshToolsFrom(bus, someSeats(), resolving)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ReadMeshToolsFrom(viewStates{r}, someSeats(), resolving)
|
||||
if err != nil || !reflect.DeepEqual(got, want) || len(got.Replacements) == 0 {
|
||||
t.Fatalf("from the views: %+v, %v\nfrom the state: %+v", got, err, want)
|
||||
}
|
||||
// A reader not yet answered says so, and reads nothing.
|
||||
if _, err := ReadMeshToolsFrom(viewStates{newMeshViews()}, someSeats(), resolving); err == nil || !strings.Contains(err.Error(), "no first view") {
|
||||
t.Fatalf("an unanswered first view: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126 B13 AC1 (ADR 0590): over the seats the mesh defines, the rendered instructions, their table among them,
|
||||
// name no address of the controller, which is not placed: each job is named by the seat that owns it now.
|
||||
func TestTheRenderedInstructionsNameNoControllerAddress(t *testing.T) {
|
||||
m, err := ReadMeshToolsFrom(aMesh(t), catalogue.Seats(), resolving)
|
||||
m, err := ReadMeshToolsFrom(aMesh(t), someSeats(), resolving)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -207,3 +284,22 @@ func TestTheRenderedInstructionsNameNoControllerAddress(t *testing.T) {
|
||||
t.Errorf("the instructions name the controller:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
// listBus answers the seat mesh-catalog's seat list from the definitions it is given.
|
||||
type listBus map[string]seatdef.Definition
|
||||
|
||||
func (b listBus) Ask(key string, body any) (json.RawMessage, error) {
|
||||
if q, _ := body.(map[string]any); q != nil && q["seat"] != nil {
|
||||
d := b[q["seat"].(string)]
|
||||
d.Format, d.Grain, d.Decision = seatdef.Format, seatdef.SeatList, "ADR 0635"
|
||||
raw, _ := json.Marshal(d)
|
||||
return json.Marshal(seatlist.Answer{Seat: d.Seat, Found: true, Entry: &seatlist.Entry{Seat: d.Seat, Source: "trunk", Digest: "d-" + d.Seat, Definition: raw}})
|
||||
}
|
||||
var l seatlist.Listing
|
||||
for n := range b {
|
||||
l.Seats = append(l.Seats, seatlist.Entry{Seat: n, Source: "trunk", Digest: "d-" + n})
|
||||
}
|
||||
return json.Marshal(l)
|
||||
}
|
||||
|
||||
func (listBus) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
|
||||
|
||||
@@ -9,5 +9,5 @@ func instructionsText(node, role string) string {
|
||||
node +
|
||||
"`\n- **Role:** " +
|
||||
role +
|
||||
"\n- The other nodes, their roles and what runs where: ask the placement seat (`mesh-placement.placement`),\n or `mesh_overview` and `mesh_machine`. Nothing here lists them, because a copy drifts.\n\n## How a session on this mesh works\n\nThe console is the only way to the mesh: the MCP server named `mesh`. It offers five tools, and\neverything else is an address you find and call through them:\n\n- `mesh_search` — words in, matching addresses out. `mesh_describe` — one address's arguments.\n- `mesh_call` — call an address. A seat the mesh holds once is `<seat>.<verb>` (the mesh's own seats\n include `mesh-placement`, `mesh-delivery`, `mesh-config` and `mesh-conditions`);\n a module on a machine is `<node>/<module>.<tool>`.\n- `mesh_overview` and `mesh_machine` — the mesh's seats and machines, and what one machine runs.\n\n- **Symptom first.** For an error, a failing service or anything unexpected, search the record with the\n literal text before forming a hypothesis: the records module's `records_search`, then\n `records_read`.\n- **Ask the mesh before changing it**, and change it through the seats' verbs or the catalog.\n- **A licence** through the `anthropic-licence-manager` seat's verbs. Never edit the agent's credentials\n file by hand, never print or ask for a token.\n\n## Hard rules\n\n- A file the mesh manages is changed through the verb or the catalog that owns it, never on disk. If\n unsure, `<node>/node-engine.composition` for the module says what the mesh writes there.\n- Never write to a store's database by hand; schema changes are numbered migrations.\n- Never push to a main branch: a branch, a pull request, and a human approval for every merge.\n- The mesh creates no symlinks, and nobody else does either.\n- A package is declared in a module, never installed by hand.\n\n## Conventions\n\n- Commit messages are concise, in the imperative, about why.\n- Test before pushing: nodes update unattended.\n- The playbooks in the record say how research, decisions, designs, issues and hand-offs are done.\n"
|
||||
"\n- The other nodes, their roles and what runs where: ask the seat `mesh-module-assigner` (`mesh-module-assigner.assignments`),\n or `mesh_overview` and `mesh_machine`. Nothing here lists them, because a copy drifts.\n\n## How a session on this mesh works\n\nThe console is the only way to the mesh: the MCP server named `mesh`. It offers five tools, and\neverything else is an address you find and call through them:\n\n- `mesh_search` — words in, matching addresses out. `mesh_describe` — one address's arguments.\n- `mesh_call` — call an address. A seat the mesh holds once is `<seat>.<verb>` (the mesh's own seats\n include `mesh-module-assigner`, `mesh-delivery`, `mesh-config` and `mesh-conditions`);\n a module on a machine is `<node>/<module>.<tool>`.\n- `mesh_overview` and `mesh_machine` — the mesh's seats and machines, and what one machine runs.\n\n- **Symptom first.** For an error, a failing service or anything unexpected, search the record with the\n literal text before forming a hypothesis: the records module's `records_search`, then\n `records_read`.\n- **Ask the mesh before changing it**, and change it through the seats' verbs or the catalog.\n- **A licence** through the `anthropic-licence-manager` seat's verbs. Never edit the agent's credentials\n file by hand, never print or ask for a token.\n\n## Hard rules\n\n- A file the mesh manages is changed through the verb or the catalog that owns it, never on disk. If\n unsure, `<node>/node-engine.composition` for the module says what the mesh writes there.\n- Never write to a store's database by hand; schema changes are numbered migrations.\n- Never push to a main branch: a branch, a pull request, and a human approval for every merge.\n- The mesh creates no symlinks, and nobody else does either.\n- A package is declared in a module, never installed by hand.\n\n## Conventions\n\n- Commit messages are concise, in the imperative, about why.\n- Test before pushing: nodes update unattended.\n- The playbooks in the record say how research, decisions, designs, issues and hand-offs are done.\n"
|
||||
}
|
||||
|
||||
@@ -71,6 +71,17 @@ type Judged struct {
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
}
|
||||
|
||||
// JudgedEvent is the body of `judged`: one per judgement asked with a bound past JudgeBound, said when it ends, with
|
||||
// its answer or why there is none. The question and the schema are never in it.
|
||||
type JudgedEvent struct {
|
||||
Ticket string `json:"ticket"`
|
||||
Node string `json:"node"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
Answer json.RawMessage `json:"answer,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// running is one judgement asked with a bound past JudgeBound, until it is collected.
|
||||
type running struct {
|
||||
done chan struct{}
|
||||
@@ -92,6 +103,9 @@ type Judge struct {
|
||||
// Wait is how long a call waits for a long judgement before it answers its ticket; JudgeWait unless a test
|
||||
// says less.
|
||||
Wait time.Duration
|
||||
// Say tells the bus a ticketed judgement ended: the event `judged`, once, under the message id it is given so a
|
||||
// repeat is dropped. Nil says nothing (a test's judge); the ticket is collected by claude_code_judged all the same.
|
||||
Say func(body JudgedEvent, id string) error
|
||||
turn chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
@@ -208,10 +222,28 @@ func (j *Judge) start(schema, model, question string, bound time.Duration) (*Jud
|
||||
go func() {
|
||||
r.out, r.err = j.run(schema, model, question, bound)
|
||||
close(r.done)
|
||||
j.said(ticket, r)
|
||||
}()
|
||||
return j.Collect(ticket)
|
||||
}
|
||||
|
||||
// said tells the bus that the judgement under a ticket ended (the event `judged`), once. A failure to say is logged: the
|
||||
// ticket stays collectible by claude_code_judged, and the asker's own bound ends its wait.
|
||||
func (j *Judge) said(ticket string, r *running) {
|
||||
if j.Say == nil {
|
||||
return
|
||||
}
|
||||
e := JudgedEvent{Ticket: ticket, Node: j.P.Node}
|
||||
if r.err != nil {
|
||||
e.Error = r.err.Error()
|
||||
} else {
|
||||
e.Model, e.Seconds, e.Answer = r.out.Model, r.out.Seconds, r.out.Answer
|
||||
}
|
||||
if err := j.Say(e, "judged-"+j.P.Node+"-"+ticket); err != nil {
|
||||
say("the judgement under ticket %s ended and could not be said on the bus: %v", ticket, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect waits as long as a call may for the judgement under a ticket: its answer, why it failed, or that it is
|
||||
// still pending. A ticket is answered once; an answer nobody collects is kept JudgedKept past its bound.
|
||||
func (j *Judge) Collect(ticket string) (*Judged, error) {
|
||||
|
||||
@@ -182,6 +182,9 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
}
|
||||
})
|
||||
j.Wait = 50 * time.Millisecond
|
||||
said := make(chan JudgedEvent, 4)
|
||||
ids := make(chan string, 4)
|
||||
j.Say = func(e JudgedEvent, id string) error { said <- e; ids <- id; return nil }
|
||||
start := time.Now()
|
||||
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
||||
if err != nil {
|
||||
@@ -203,6 +206,21 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// One `judged` per ticket, with the answer, under an id of the ticket's own; none for a judgement with no ticket.
|
||||
select {
|
||||
case e := <-said:
|
||||
if e.Ticket != out.Ticket || e.Node != "control" || string(e.Answer) != `{"verdict":"none"}` || e.Error != "" || e.Model != "sonnet" {
|
||||
t.Fatalf("judged: %+v", e)
|
||||
}
|
||||
if id := <-ids; id != "judged-control-"+out.Ticket {
|
||||
t.Fatalf("judged under %q", id)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the ended judgement was not said")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("judged said twice")
|
||||
}
|
||||
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
||||
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
||||
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
||||
@@ -224,10 +242,23 @@ func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
||||
}
|
||||
j.Wait = 20 * time.Millisecond
|
||||
j.Bound = time.Second // the test's stand-in for JudgeBound: within is past it, so a ticket; far past Wait on a loaded machine
|
||||
failed := make(chan JudgedEvent, 1)
|
||||
j.Say = func(e JudgedEvent, _ string) error { failed <- e; return nil }
|
||||
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 2 * time.Second})
|
||||
if err != nil || !out.Pending {
|
||||
t.Fatalf("%+v %v", out, err)
|
||||
}
|
||||
defer func() {
|
||||
// A failed judgement is said too, with why and no answer.
|
||||
select {
|
||||
case e := <-failed:
|
||||
if e.Ticket != out.Ticket || e.Answer != nil || !strings.Contains(e.Error, "did not answer within") {
|
||||
t.Errorf("judged: %+v", e)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the failed judgement was not said")
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 3000; i++ {
|
||||
got, err := j.Collect(out.Ticket)
|
||||
if err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -21,9 +22,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
func say(format string, args ...any) {
|
||||
@@ -180,12 +184,12 @@ func (s memoryStateOf) Delete(key string) error { return s.s.Delete(key) }
|
||||
var memoryCarrier *MemoryCarrier
|
||||
|
||||
// sayCondition says a condition on the operator-channel seat under this module's name (novox/hq ADR 0437 §9).
|
||||
func sayCondition(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject("claude-code")
|
||||
func sayCondition(c oc.Condition) error {
|
||||
subject, err := occ.ConditionSubject("claude-code")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := asks.EncodeCondition(c)
|
||||
raw, err := occ.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -205,23 +209,53 @@ func (s stateOf) Get(key string) (json.RawMessage, bool, error) {
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
// nodesRunningMe is the nodes claude-code runs on, from the seat mesh-placement's placement — for the register
|
||||
// tool's question.
|
||||
func nodesRunningMe() ([]string, error) { return NodesRunning(busStates{}, "claude-code") }
|
||||
// nodesRunningMe is the nodes claude-code runs on, from the assigner's assignments — for the register tool's question.
|
||||
func nodesRunningMe() ([]string, error) { return NodesRunning(viewStates{meshViews}, "claude-code") }
|
||||
|
||||
// busStates reads the states the manifest declares in its reads through the runtime.
|
||||
type busStates struct{}
|
||||
// The owners' read tools of the states this module follows, which the manifest invokes (novox/hq ADR 0615 §1).
|
||||
// The holders whose views this module follows, as its manifest's `consumes` and `invokes` name them (the manifest test
|
||||
// holds them equal).
|
||||
const (
|
||||
assignerModule = "mesh-delivery-assigner"
|
||||
indexerModule = "mesh-delivery-indexer"
|
||||
)
|
||||
|
||||
func (busStates) Keys(read string) ([]string, error) { return stdio.State(read).Keys() }
|
||||
const (
|
||||
ModulesTool = indexerModule + ".indexer_read_modules"
|
||||
ManifestsTool = indexerModule + ".indexer_read_manifests"
|
||||
AssignmentsTool = assignerModule + ".assigner_read_assignments"
|
||||
)
|
||||
|
||||
func (busStates) Get(read, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(read).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
// meshViews is this module's picture of the indexer's modules and manifests and the assigner's assignments: each asked
|
||||
// once of the owner's read tool, then followed on the owner's views family. It reads none of their state.
|
||||
var meshViews *views.Reader
|
||||
|
||||
// meshChanged is told, without waiting, that a followed state changed: the instructions' table is read again.
|
||||
var meshChanged = make(chan struct{}, 1)
|
||||
|
||||
// newMeshViews builds the reader of those three states; Start asks them.
|
||||
func newMeshViews() *views.Reader {
|
||||
v := &views.Reader{Module: Module, Logf: func(f string, a ...any) { say(f, a...) }, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")},
|
||||
Acts: handActs{},
|
||||
Changed: func(string, string, json.RawMessage, bool) {
|
||||
select {
|
||||
case meshChanged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}}
|
||||
v.Follow(indexerModule, "modules", ModulesTool)
|
||||
v.Follow(indexerModule, "manifests", ManifestsTool)
|
||||
v.Follow(assignerModule, "assignments", AssignmentsTool)
|
||||
return v
|
||||
}
|
||||
|
||||
// viewStates reads the followed states from the reader's pictures, never from the owners' state.
|
||||
type viewStates struct{ v *views.Reader }
|
||||
|
||||
func (s viewStates) Keys(read string) ([]string, error) { return s.v.Keys(read) }
|
||||
|
||||
func (s viewStates) Get(read, key string) ([]byte, bool, error) { return s.v.Get(read, key) }
|
||||
|
||||
func fingerprintOfFile(path string) any {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -291,7 +325,7 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
|
||||
Description: "Write Claude Code's managed directory now, from the mesh's facts, this module's settings and the servers registered here — after reading the bus again for what each tool replaces and which machines the mesh has (the instructions' table and the shell's guard).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
answer := map[string]any{}
|
||||
if m, err := ReadMeshToolsFrom(busStates{}, catalogue.Seats(), lookup); err != nil {
|
||||
if m, err := ReadMeshToolsFrom(viewStates{meshViews}, definedSeats(), lookup); err != nil {
|
||||
answer["asked"] = "the bus could not be read in full (" + err.Error() + "); what was kept is rendered"
|
||||
} else {
|
||||
KeepMeshTools(p, m)
|
||||
@@ -372,7 +406,13 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
|
||||
out = append(out, proposalTools(p)...)
|
||||
out = append(out, waitTools(p)...)
|
||||
judge := NewJudge(p)
|
||||
// The event `judged`, one per ticket when its judgement ends: the asker hears it instead of asking again.
|
||||
judge.Say = func(e JudgedEvent, id string) error {
|
||||
return stdio.Emit("judged", e, stdio.EmitOptions{MessageID: id})
|
||||
}
|
||||
out = append(out, judgeTool(judge), judgedTool(judge))
|
||||
out = append(out, meshViews.Tools("claude_code")...)
|
||||
out = append(out, seatListTools()...)
|
||||
return append(out, configTools(p, config, configView)...)
|
||||
}
|
||||
|
||||
@@ -418,6 +458,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
p, launched := PathsFrom(os.Getenv)
|
||||
meshViews = newMeshViews()
|
||||
if launched {
|
||||
fsys = FilesFor(p)
|
||||
}
|
||||
@@ -447,6 +488,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
meshViews.Start(context.Background())
|
||||
go refreshMeshTools(p)
|
||||
servers := stateOf{stdio.State("servers")}
|
||||
view := NewServerView(p)
|
||||
@@ -624,3 +666,27 @@ func refusals(n int) string {
|
||||
}
|
||||
return fmt.Sprintf(" (after %d refusal(s))", n)
|
||||
}
|
||||
|
||||
// seatListAgainCallers may read the seat list again: every operator's session and the healers (ADR 0636 §7).
|
||||
var seatListAgainCallers = []string{stdio.AnyPerson, "mesh-healers"}
|
||||
|
||||
// seatListTools are the seat list reader's two tools: what the list holds and what it waits for, and a read now. The
|
||||
// reader starts once the runtime serves, after the tools are listed, so each tool finds it when it is called.
|
||||
func seatListTools() []stdio.Tool {
|
||||
names := []string{"claude_code_seat_list", "claude_code_seat_list_again"}
|
||||
callers := seatListAgainCallers
|
||||
shape := seatlist.Tools(&seatlist.Reader{}, names[0], names[1], callers)
|
||||
for i := range shape {
|
||||
run := i
|
||||
shape[i].Run = func(args map[string]any) (any, error) {
|
||||
seatListMu.Lock()
|
||||
r := seatList
|
||||
seatListMu.Unlock()
|
||||
if r == nil {
|
||||
return nil, errors.New("the seat list reader has not started yet: the runtime has not served this module")
|
||||
}
|
||||
return seatlist.Tools(r, names[0], names[1], callers)[run].Run(args)
|
||||
}
|
||||
}
|
||||
return shape
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user